def DbBuildInvertOld(ele,invfunc): raise Exception('Deprecated') # Check ele if ele not in ELELIST['maps']: raise Exception('Invalid element') # Target inv db dbfileinv = ele.upper()+'_INV.db' # Lock and open inv db lock = FileLock(dbfileinv,5) lock.acquire() dbinv = anydbm.open(dbfileinv,'c') # Clear inv db dbinv.clear() # List dir for mapdbfile in os.listdir('maps'): mapid = mapdbfile[:-3] dbmap = anydbm.open('maps/%s' % mapdbfile,'r') if dbmap.has_key(ele): value = dbmap[ele] for word in invfunc(value): if dbinv.has_key(word): dbinv[word] = dbinv[word] + (',%s' % mapid) else: dbinv[word] = '%s' % mapid dbmap.close() dbinv.close() lock.release() # Rebuild is no more needed RearmRebuild(ele)
def DbBuildInvert(dbtype,ele,invfunc): if dbtype not in DBTYPES: raise Exception('Invalid database type') # Check ele if ele not in ELELIST[dbtype]: raise Exception('Invalid element') #print '<!-- DbBuildInvert -->\n' # Target inv db dbfileinv = ele.upper()+'_INV.db' # Lock and open inv db lock = FileLock(dbfileinv,5) lock.acquire() #Log('DbBuildInvert open db c %s\n'%dbfileinv) dbinv = anydbm.open(dbfileinv,'c') # Clear inv db dbinv.clear() # List dir for dbfile in os.listdir(dbtype): id = dbfile[:-3] #Log('DbBuildInvert open db r %s/%s\n'%(dbtype,dbfile)) db = anydbm.open('%s/%s' % (dbtype,dbfile),'r') if db.has_key(ele): value = db[ele] for word in invfunc(value): if dbinv.has_key(word): dbinv[word] = dbinv[word] + (',%s' % id) else: dbinv[word] = '%s' % id db.close() #Log('DbBuildInvert close db r %s/%s\n'%(dbtype,dbfile)) dbinv.close() #Log('DbBuildInvert close db c %s\n'%dbfileinv) lock.release() # Rebuild is no more needed RearmRebuild(ele)
def DbAddComment(mapid,user,comment): mapfile = 'maps/%s.db' % mapid if not os.access(mapfile,os.F_OK): raise Exception('Invalid map id %s' % mapid) d = getCurrentDate() lock = FileLock(mapfile,5) lock.acquire() #Log('DbAddComment open db r %s\n' % mapfile) db = anydbm.open(mapfile,'r') if db.has_key('last_comment_id'): last_comment_id = int(db['last_comment_id']) else: last_comment_id = 0 db.close() #Log('DbAddComment close db r %s\n' % mapfile) last_comment_id += 1 if last_comment_id>99999: lock.release() raise Exception('Max comments reached') #Log('DbAddComment open db c %s\n' % mapfile) db = anydbm.open(mapfile,'c') db['last_comment_id'] = str(last_comment_id) db['comment%.5d'%last_comment_id] = '%s,%s,%s' % (d,user,comment) db.close() #Log('DbAddComment close db c %s\n' % mapfile) lock.release()
def run(self): while True: lock = FileLock("/var/lock/baseDaemon.lock") #Just to be safe; pulling data shouldn't take more than 2h lock.acquire() wikiDatesToMongo(False) time.sleep(2*60*60) lock.release()
def update(self, lock=True): if lock: flock = FileLock(self.inifile) flock.acquire() try: inifp = open(self.inifile, 'w') self.cfg.write(inifp) inifp.close() if lock: flock.release() return True except: if lock: flock.release() return False
def runjobs(self, verbose = None): """ Run the jobs """ # The verbose flag if verbose is not None: self.verbose = verbose # Find all 'job_' methods for (exc_name, exc_value) in inspect.getmembers(self, lambda x: inspect.ismethod(x)): # The method's name must start with uppercase if exc_name[0].isupper(): lock = FileLock(os.path.join('/', 'tmp', 'TranPy-%s-%s.lock' % (self.job_class, exc_name))) # Try to get the lock if lock.acquire(): if self.verbose: print >>sys.stderr, 'Running %s %s' % (self.job_class, exc_name) # Run the job try: exc_value() except: if self.verbose: traceback.print_exc(file = sys.stderr) finally: # Release the lock lock.release() else: if self.verbose: print >>sys.stderr, 'Locked %s %s' % (self.job_class, exc_name)
def __init__(self, i, model_cls, batch_size, all_reduce_alg=None, num_devices=1, use_cpus=False, max_bytes=0, use_xray=True, plasma_op=False, verbose=False): # TODO - just port VariableMgrLocalReplicated if use_xray: if num_devices == 4: gpu0 = FileLock("/tmp/gpu0") gpu1 = FileLock("/tmp/gpu1") try: gpu0.acquire(timeout=0) os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3" except: gpu1.acquire(timeout=0) os.environ["CUDA_VISIBLE_DEVICES"] = "4,5,6,7" else: os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3,4,5,6,7" print("CUDA VISIBLES", os.environ["CUDA_VISIBLE_DEVICES"]) self.i = i assert num_devices > 0 tf_session_args = { "device_count": { "CPU": num_devices }, "log_device_placement": False, "gpu_options": tf.GPUOptions(force_gpu_compatible=True), "inter_op_parallelism_threads": 128, } config_proto = tf.ConfigProto(**tf_session_args) self.sess = tf.Session(config=config_proto) models = [] grad_ops = [] self.iter = 0 if use_cpus: device_tmpl = "/cpu:%d" else: device_tmpl = "/gpu:%d" for device_idx in range(num_devices): device = device_tmpl % device_idx with tf.device(device): with tf.variable_scope("device_%d" % device_idx): print("DEVICE: ", device) model = model_cls(batch=batch_size, use_cpus=use_cpus, device=device) models += [model] model.grads = [ t for t in model.optimizer.compute_gradients(model.loss) if t[0] is not None ] grad_ops.append(model.grads) self.models = models if num_devices == 1: assert not max_bytes, "Not supported with 1 GPU" self.packed_grads_and_vars = grad_ops elif all_reduce_alg: if max_bytes: from tfbench import modified_allreduce self.packed_grads_and_vars, packing_vals = modified_allreduce.sum_gradients_all_reduce( "", grad_ops, 1, all_reduce_alg, 1, list(range(num_devices)), agg_small_grads_max_bytes=max_bytes) else: self.packed_grads_and_vars = allreduce.sum_gradients_all_reduce( "", grad_ops, 1, all_reduce_alg, 1, list(range(num_devices))) self.per_device_grads = [ list(zip(*dev_gv))[0] for dev_gv in self.packed_grads_and_vars ] assert (len(self.per_device_grads) == num_devices) self.num_grads = num_grads = len(self.packed_grads_and_vars[0]) if max_bytes: assert (num_grads < 314) print("Packed grads => {} tensors".format(num_grads)) else: assert (num_grads == 314) # Ops for reading grads with the right control deps nccl_noops = [] for j in range(num_grads)[::-1]: with tf.control_dependencies( nccl_noops + [dev_grad[j] for dev_grad in self.per_device_grads]): nccl_noops = [tf.no_op()] # You must fetch this otherwise the NCCL allreduce will hang self.nccl_control_out = tf.group(*nccl_noops) round_robin_devices = False if args.plasma_op: memcpy_plasma_module = tf.load_op_library( "/home/ubuntu/osdi2018/ops/memcpy_plasma_op.so") # For fetching grads -> plasma self.plasma_in_grads = [] self.plasma_in_grads_oids = [ tf.placeholder(shape=[], dtype=tf.string) for _ in range(num_grads) ] ix = 0 for j in range(num_grads): grad = self.per_device_grads[ix][j] if round_robin_devices: ix += 1 # round robin assignment ix %= num_devices with tf.device(self.models[ix].device): plasma_grad = memcpy_plasma_module.tensor_to_plasma( [grad], self.plasma_in_grads_oids[j], plasma_store_socket_name=ray.worker.global_worker. plasma_client.store_socket_name, plasma_manager_socket_name=ray.worker.global_worker. plasma_client.manager_socket_name) self.plasma_in_grads.append(plasma_grad) # For applying grads <- plasma unpacked_gv = [] self.plasma_out_grads_oids = [ tf.placeholder(shape=[], dtype=tf.string) for _ in range(num_grads) ] packed_plasma_grads = [] ix = 0 for j in range(num_grads): with tf.device(self.plasma_in_grads[j].device): with tf.control_dependencies([self.plasma_in_grads[j]]): grad_ph = memcpy_plasma_module.plasma_to_tensor( self.plasma_out_grads_oids[j], plasma_store_socket_name=ray.worker.global_worker. plasma_client.store_socket_name, plasma_manager_socket_name=ray.worker. global_worker.plasma_client.manager_socket_name) grad_ph = tf.reshape(grad_ph, self.packed_grads_and_vars[0][j][0].shape) print("Packed tensor", grad_ph) packed_plasma_grads.append(grad_ph) for i in range(num_devices): per_device = [] for j, (g, v) in enumerate(self.packed_grads_and_vars[i]): grad_ph = packed_plasma_grads[j] per_device.append((grad_ph, v)) unpacked_gv.append(per_device) if max_bytes: unpacked_gv = allreduce.unpack_small_tensors( unpacked_gv, packing_vals) elif max_bytes: unpacked_gv = allreduce.unpack_small_tensors( self.packed_grads_and_vars, packing_vals) else: unpacked_gv = self.packed_grads_and_vars # Same shape as packed_grads_and_vars assert len(unpacked_gv) == num_devices assert len(unpacked_gv[0]) == 314 assert len(unpacked_gv[0][0]) == 2 apply_ops = [] to_apply = unpacked_gv[0] for ix, m in enumerate(models): apply_ops.append( m.optimizer.apply_gradients([ (g, v) for ((g, _), (_, v)) in zip(to_apply, unpacked_gv[ix]) ])) self.apply_op = tf.group(*apply_ops) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) self.sess.run(init_op)
def main(): ret = SUCCESS_EXITVAL parser = argparse.ArgumentParser(description='project mirroring', parents=[get_base_parser( tool_version=__version__) ]) parser.add_argument('project', nargs='*', default=None) parser.add_argument('-a', '--all', action='store_true', help='mirror all indexed projects', default=False) parser.add_argument('-c', '--config', help='config file in JSON/YAML format') parser.add_argument('-U', '--uri', default='http://localhost:8080/source', help='uri of the webapp with context path') parser.add_argument('-b', '--batch', action='store_true', help='batch mode - will log into a file') parser.add_argument('-B', '--backupcount', default=8, help='how many log files to keep around in batch mode') parser.add_argument('-I', '--check-changes', action='store_true', help='Check for changes in the project or its' ' repositories,' ' terminate the processing' ' if no change is found.') parser.add_argument('-w', '--workers', default=cpu_count(), help='Number of worker processes') try: args = parser.parse_args() except ValueError as e: return fatal(e, False) logger = get_console_logger(get_class_basename(), args.loglevel) if len(args.project) > 0 and args.all: return fatal("Cannot use both project list and -a/--all", False) if not args.all and len(args.project) == 0: return fatal("Need at least one project or --all", False) if args.config: config = read_config(logger, args.config) if config is None: return fatal("Cannot read config file from {}". format(args.config), False) else: config = {} uri = args.uri if not is_web_uri(uri): return fatal("Not a URI: {}".format(uri), False) logger.debug("web application URI = {}".format(uri)) if not check_configuration(config): return 1 # Save the source root to avoid querying the web application. source_root = get_config_value(logger, 'sourceRoot', uri) if not source_root: return 1 logger.debug("Source root = {}".format(source_root)) hookdir = config.get(HOOKDIR_PROPERTY) if hookdir: logger.debug("Hook directory = {}".format(hookdir)) command_timeout = get_int(logger, "command timeout", config.get(CMD_TIMEOUT_PROPERTY)) if command_timeout: logger.debug("Global command timeout = {}".format(command_timeout)) hook_timeout = get_int(logger, "hook timeout", config.get(HOOK_TIMEOUT_PROPERTY)) if hook_timeout: logger.debug("Global hook timeout = {}".format(hook_timeout)) logdir = None # Log messages to dedicated log file if running in batch mode. if args.batch: logdir = config.get(LOGDIR_PROPERTY) if not logdir: return fatal("The {} property is required in batch mode". format(LOGDIR_PROPERTY), False) projects = args.project if len(projects) == 1: lockfile = projects[0] + "-mirror" else: lockfile = os.path.basename(sys.argv[0]) if args.all: projects = list_indexed_projects(logger, args.uri) lock = FileLock(os.path.join(tempfile.gettempdir(), lockfile + ".lock")) try: with lock.acquire(timeout=0): with Pool(processes=int(args.workers)) as pool: worker_args = [] for x in projects: worker_args.append([x, logdir, args.loglevel, args.backupcount, config, args.check_changes, args.uri, source_root, args.batch]) try: project_results = pool.map(worker, worker_args, 1) except KeyboardInterrupt: return FAILURE_EXITVAL else: if any([x == FAILURE_EXITVAL for x in project_results]): ret = FAILURE_EXITVAL if all([x == CONTINUE_EXITVAL for x in project_results]): ret = CONTINUE_EXITVAL except Timeout: logger.warning("Already running, exiting.") return FAILURE_EXITVAL logging.shutdown() return ret
class EngineService: """Service that provides methods for interacting with BAS engine.""" _exe_dir: str = None """The path to the directory in which the executable file of the engine is located.""" _zip_dir: str = None """The path to the directory in which the archive file of the engine is located.""" def __init__(self, client): """Create an instance of EngineService class.""" script_name = client.options.script_name working_dir = client.options.working_dir self._loop = client.loop self._script_dir = path.join(working_dir, "run", script_name) self._engine_dir = path.join(working_dir, "engine") self._script_name = script_name self._process = None self._lock = None async def start(self, port: int) -> None: """Asynchronously start the engine service with the specified port. Arguments: port (int): Selected port number. """ arch = 64 if machine().endswith('64') else 32 zip_name = f'FastExecuteScriptProtected.x{arch}' url_name = f'FastExecuteScriptProtected{arch}' zip_path = path.join(self._zip_dir, f'{zip_name}.zip') if not path.exists(self._zip_dir): makedirs(self._zip_dir) await self._download_executable(zip_path, zip_name, url_name) if not path.exists(self._exe_dir): makedirs(self._exe_dir) await self._extract_executable(zip_path) self._start_engine_process(port) self._clear_run_directory() async def initialize(self): url = f'{ENDPOINT}/scripts/{self._script_name}/properties' async with ClientSession(loop=self._loop) as session: async with session.get(url) as response: script = Script(await response.json()) if not script.is_exist: raise ScriptNotExistError() if not script.is_supported: raise ScriptNotSupportedError() self._zip_dir = path.join(self._engine_dir, script.engine_version) self._exe_dir = path.join(self._script_dir, script.hash[0:5]) async def _download_executable(self, zip_path: str, zip_name: str, url_name: str) -> None: url = f"{ENDPOINT}/distr/{url_name}/{path.basename(self._zip_dir)}/{zip_name}.zip" async with ClientSession(loop=self._loop) as session: async with session.get(url) as response: async with open(zip_path, 'wb') as file: while True: chunk = await response.content.read(1024 * 16) if not chunk: break await file.write(chunk) return await response.release() async def _extract_executable(self, zip_path: str) -> None: with ZipFile(zip_path, 'r') as file: async def task(name, zip_file: ZipFile): zip_file.extract(name, self._exe_dir, None) tasks = [task(name, file) for name in file.namelist()] await asyncio.wait(tasks, loop=self._loop) def _start_engine_process(self, port: int) -> None: self._process = subprocess.Popen([ path.join(self._exe_dir, 'FastExecuteScript.exe'), f'--remote-control-port={port}', f'--remote-control' ], cwd=self._exe_dir) lock = self._get_lock_path() self._lock = FileLock(lock) self._lock.acquire() def _clear_run_directory(self) -> None: for dir_path in [ name for name in listdir(self._script_dir) if path.isfile(name) ]: dir_path = path.join(self._script_dir, dir_path) lock_path = self._get_lock_path(dir_path) if not is_locked(lock_path): rmtree(dir_path) def _get_lock_path(self, dir_path=None) -> str: return path.join(dir_path or self._exe_dir, '.lock') async def close(self) -> None: """Close the engine service.""" self._process.kill() self._lock.release()
def read( self, file_path: Union[Path, str] ) -> Union[AllennlpDataset, AllennlpLazyDataset]: """ Returns an dataset containing all the instances that can be read from the file path. If `self.lazy` is `False`, this eagerly reads all instances from `self._read()` and returns an `AllennlpDataset`. If `self.lazy` is `True`, this returns an `AllennlpLazyDataset`, which internally relies on the generator created from `self._read()` to lazily produce `Instance`s. In this case your implementation of `_read()` must also be lazy (that is, not load all instances into memory at once), otherwise you will get a `ConfigurationError`. In either case, the returned `Iterable` can be iterated over multiple times. It's unlikely you want to override this function, but if you do your result should likewise be repeatedly iterable. """ if not isinstance(file_path, str): file_path = str(file_path) lazy = getattr(self, "lazy", None) if lazy is None: warnings.warn( "DatasetReader.lazy is not set, " "did you forget to call the superclass constructor?", UserWarning, ) if lazy: return AllennlpLazyDataset(self._instance_iterator, file_path) else: cache_file: Optional[str] = None if self._cache_directory: cache_file = self._get_cache_location_for_file_path(file_path) if cache_file is not None and os.path.exists(cache_file): try: # Try to acquire a lock just to make sure another process isn't in the middle # of writing to the cache. cache_file_lock = FileLock( cache_file + ".lock", timeout=self.CACHE_FILE_LOCK_TIMEOUT) cache_file_lock.acquire() # We make an assumption here that if we can obtain the lock, no one will # be trying to write to the file anymore, so it should be safe to release the lock # before reading so that other processes can also read from it. cache_file_lock.release() logger.info("Reading instances from cache %s", cache_file) instances = self._instances_from_cache_file(cache_file) except Timeout: logger.warning( "Failed to acquire lock on dataset cache file within %d seconds. " "Cannot use cache to read instances.", self.CACHE_FILE_LOCK_TIMEOUT, ) instances = self._multi_worker_islice( self._read(file_path)) else: instances = self._multi_worker_islice(self._read(file_path)) # Then some validation. if not isinstance(instances, list): instances = list(instances) if not instances: raise ConfigurationError( "No instances were read from the given filepath {}. " "Is the path correct?".format(file_path)) # And finally we try writing to the cache. if cache_file is not None and not os.path.exists(cache_file): if self.max_instances is not None: # But we don't write to the cache when max_instances is specified. logger.warning( "Skipping writing to data cache since max_instances was specified." ) elif util.is_distributed() or (get_worker_info() and get_worker_info().num_workers): # We also shouldn't write to the cache if there's more than one process loading # instances since each worker only receives a partial share of the instances. logger.warning( "Can't cache data instances when there are multiple processes loading data" ) else: try: with FileLock(cache_file + ".lock", timeout=self.CACHE_FILE_LOCK_TIMEOUT): self._instances_to_cache_file( cache_file, instances) except Timeout: logger.warning( "Failed to acquire lock on dataset cache file within %d seconds. " "Cannot write to cache.", self.CACHE_FILE_LOCK_TIMEOUT, ) return AllennlpDataset(instances)
def test_actor_writer_2(workflow_start_regular, tmp_path): g_lock = str(Path(tmp_path / "g.lock")) incr_lock = str(Path(tmp_path / "incr.lock")) val_lock = str(Path(tmp_path / "val.lock")) val_err = str(Path(tmp_path / "val.err")) incr_err = str(Path(tmp_path / "incr.err")) @workflow.virtual_actor class SyncCounter: def __init__(self, val_lock: str, incr_lock: str, g_lock: str, val_err: str, incr_err: str): self.val_lock = val_lock self.incr_lock = incr_lock self.g_lock = g_lock self.val_err = val_err self.incr_err = incr_err self.v = 0 if Path(self.val_err).exists(): raise ValueError() @workflow.virtual_actor.readonly def val(self): with FileLock(self.val_lock), FileLock(self.g_lock): if Path(self.val_err).exists(): raise ValueError() return self.v def incr(self, create_incr_err=False): with FileLock(self.incr_lock), FileLock(self.g_lock): if Path(self.incr_err).exists(): raise ValueError() if create_incr_err: Path(incr_err).touch() self.v += 1 return self.v def __getstate__(self): return (self.v, self.val_lock, self.incr_lock, self.g_lock, self.val_err, self.incr_err) def __setstate__(self, state): (self.v, self.val_lock, self.incr_lock, self.g_lock, self.val_err, self.incr_err) = state # trigger error in init Path(val_err).touch() actor = SyncCounter.get_or_create("sync_counter", val_lock, incr_lock, g_lock, val_err, incr_err) with pytest.raises(Exception): actor.incr.run() Path(val_err).unlink() assert ray.get([actor.incr.run_async() for _ in range(9)]) == list(range(2, 11)) incr_lock = FileLock(incr_lock) incr_lock.acquire() objs = [actor.incr.run_async() for _ in range(10)] assert 10 == actor.val.run() Path(val_err).touch() with pytest.raises(Exception): actor.val.run() Path(val_err).unlink() incr_lock.release() assert ray.get(objs) == list(range(11, 21)) # test error cases actor.incr.run_async() # 21 actor.incr.run_async() # 22 actor.incr.run_async(create_incr_err=True) # 23 actor.incr.run_async() # 24 s5 = actor.incr.run_async() # 25 with pytest.raises(Exception): ray.get(s5) assert 23 == actor.val.run() Path(incr_err).unlink() obj = workflow.resume("sync_counter") assert 25 == ray.get(obj)[0] assert 25 == actor.val.run()
class AquariumController: def __init__(self): logger.info("=" * 125) logger.info("Initializing".center(125)) logger.info("=" * 125) self.load_data() self.load_config() self.hw_controller = Hardware() self.email = EmailAlerts() self.temp_c, self.temp_f = self.hw_controller.read_temperature("temp_tank") self.file_name = 'graph_data.csv' lock_path = self.file_name+".lock" self.lock = FileLock(lock_path) self.con = self.sql_connection() self.cursorObj = self.con.cursor() self.calibration_data = { "Co2 Calibration Data": {}, "Fertilizer Calibration Data": {}, "Water Conditioner Calibration Data": {}, } self.ratio_data = { "Tank Size": {}, "Co2 Ratio": {}, "Fertilizer Ratio": {}, "Water Conditioner Ratio": {}, } self.setting_data = { "Network": {}, "Temperature Alerts": {}, "Email Alert": {} } self.cal_status = ["Success", "Failed", "In Progress", "None"] self.network_config = { "sender_email": {}, "target_email": {}, "password_email": {}, "service_email": {}, "alert_limit": {} } self.alert_counter = {} async def start_calibration(self, pump_type: str): try: '''asyncio.create_task(self.hw_controller.notification_led_pulse()) self.hw_controller.button_state() asyncio.create_task(self.hw_controller.notification_led_flash()) self.calibrate_pump(pump_type) self.hw_controller.notification_led_stop() ''' await self.hw_controller.notification_led_pulse() self.hw_controller.button_state() await self.hw_controller.notification_led_flash() self.calibrate_pump(pump_type) self.hw_controller.notification_led_stop() except CalibrationCancelled: logger.info("!Calibration was Cancelled!") def calibrate_pump(self, pump_type): logger.info(f"Running {pump_type} Pump\n" f"{pump_type}", "Calibration started".center(125)) self.calibration_status(pump_type, self.cal_status[2]) start = time.time() self.hw_controller.pump_on(pump_type) self.hw_controller.button_state() logger.info(f"Stopping {pump_type}\n" f"{pump_type}", "Calibration finished.".center(125)) self.calibration_status(pump_type, self.cal_status[0]) end = time.time() self.hw_controller.pump_off(pump_type) cal_time = round(end - start, 2) per_ml = round(cal_time / 10, 2) logger.info(type(cal_time)) logger.info(f"{pump_type} Runtime: {cal_time}") self.calibration_data[f"{pump_type} Calibration Data"].update( { "Time per 10mL": cal_time, "Time per 1mL": per_ml } ) self.save() def stop_calibration(self, pump_type: str): self.hw_controller.stop_calibration() def cal_status(self, pump_type: str): self.hw_controller.calibration_status() def tank_temperature(self): temp_c, temp_f = self.hw_controller.read_temperature("temp_tank") try: ht = self.setting_data["Temperature Alerts"]["High Temp"] lt = self.setting_data["Temperature Alerts"]["Low Temp"] ht_checked = self.setting_data["Temperature Alerts"]["High Enabled"] lt_checked = self.setting_data["Temperature Alerts"]["Low Enabled"] if ht_checked == '2': if temp_c > float(ht): logger.critical("HIGH TEMP ALERT!!!".center(125)) cur_temp = temp_c high_temp_threshold = ht self.email.high_temp_alert_example(cur_temp, high_temp_threshold) if lt_checked == '2': if temp_c < float(lt): logger.critical("LOW TEMP ALERT!!!".center(125)) except KeyError: logger.warning("No Temperature Alert Data") return round(temp_c, 2) def email_test(self): logger.info("=" * 125) self.email.msg_format(alert_type='EMAIL TEST', variable_data=None, custom_msg=self.email.templates.test_msg()) logger.info("=" * 125) def email_ht_alert(self): logger.info("=" * 125) logger.info("HT Alert Email Function".center(125)) logger.info("=" * 125) data = { "Current Temperature": self.temp_c, "Current Threshold": self.setting_data["Temperature Alerts"]["High Temp"] } send = self.email.email_builder(alert_type='High Temperature', alert_data=data) logger.info("=" * 125) """ def alert_counters(self, alert_type): name = f"{alert_type}" + " counter" #dict = self.alert_counter[""] if name in self.alert_counter.keys(): for value in name: self.alert_counter[(alert_type + " counter")].update( { f"{name}": int(value)+1, } ) self.save_config() """ def calibration_status(self, pump_type, cal_status): logger.info(f"pump: {pump_type}, status: {cal_status}") return pump_type, cal_status def ratioequals(self, ratio_results): logger.info("ratio equals function") logger.info(f"values {ratio_results}") new_ratio = ('Tank', 'Co2_ratio', 'Co2_water', 'Fertilizer_ratio', 'Fertilizer_water', 'WaterConditioner_ratio'\ , 'WaterConditioner_water') zipratio = zip(new_ratio, ratio_results) ratiodict = dict(zipratio) for value in ['Co2', 'Fertilizer', 'WaterConditioner']: logger.info(type(value)) ratio = float(ratiodict[value + '_ratio']) water = float(ratiodict[value + '_water']) tank = float(ratiodict['Tank']) try: dosage = ratio * tank / water except ZeroDivisionError: dosage = 0 ratiodict[value + '_dosage'] = "{:.2f}".format(float(dosage)) #if dosage != 0 else 0 self.ratio_data = ratiodict logger.info(f"Dict Data: {ratiodict}") self.save() def ratios(self, ratio_results): logger.info(f"Ratio: {ratio_results}") logger.info('Tank Size: {} Litres,\n' 'Co2 Concentrate: {} mL, Co2 to Water: {} Litres,\n' 'Fertilizer Concentrate: {} mL, Fertilizer to Water: {} Litres,\n' 'WaterConditioner Concentrate: {} mL, WaterConditioner to Water: {} Litres'.format( *ratio_results)) self.ratioequals(ratio_results) def alert_data(self, ht: int, lt: int, ht_enabled, lt_enabled): logger.info("New Alert Set") logger.info(f"High Temperature: {ht} Enabled:{ht_enabled}") logger.info(f"Low Temperature: {lt} Enabled:{lt_enabled}") self.setting_data["Temperature Alerts"].update( { "High Temp": ht, "High Enabled": ht_enabled, "Low Temp": lt, "Low Enabled": lt_enabled } ) self.save() def save(self): data = { "Setting Data": self.setting_data, "Ratio Data": self.ratio_data, "Calibration Data": self.calibration_data, # "Schedule Data": schedule_data, # "Dosage Data": dosage_data, # "Light Hour Data": light_hour_data } with open('data.txt', 'w') as json_file: json_file.write(json.dumps(data, indent=4)) logger.info("Settings Updated") def save_config(self): logger.info(f"Before updating config: {self.network_config}") config_data = { "network_config": self.network_config, "alert_counters": self.alert_counter, } try: with open('config.json', 'w') as json_data_file: json_data_file.write(json.dumps(config_data, indent=4)) logger.info(f"Email Details Saved") except: logger.exception(f" Email Details not Saved") logger.info(f"After updating config_data: {config_data}") logger.info(f"After updating config: {self.network_config}") def save_email(self, email_user: str, service_email: str, password_email): logger.info("=" * 125) logger.info(f"Email Address Updated".center(125)) logger.info("=" * 125) if "@" not in email_user: logger.info(f"adding {service_email} to {email_user}") email_user = email_user.strip() + service_email.strip() else: logger.info(f"Email already has '@' ") self.network_config.update( { "sender_email": "*****@*****.**", "target_email": email_user.strip(), "password_email": password_email, "service_email": service_email.strip(), #"alert_limit": alert_limit, } ) logger.info(f"{email_user}") logger.info(f"Email Pass: {password_email}") #logger.info(f"Alert Limit: {alert_limit} Per Day") self.save_config() logger.info("=" * 125) def saveEmail_limit(self, alert_limit: int): logger.info("=" * 125) logger.info(f"Email Alert Limit Updated".center(125)) logger.info("=" * 125) self.network_config.update( { "alert_limit": alert_limit, } ) logger.info(f"Alert Limit: {alert_limit} Per Day") self.save_config() logger.info("=" * 125) def load_data(self): logger.info("=" * 125) logger.info("Loading 'data.txt' From Local Path") try: if os.path.isfile('data.txt'): with open('data.txt', 'r') as json_file: data = json.loads(json_file.read()) logger.success("'data.txt' Loaded") logger.debug(f"'data.txt' contents: {data}") logger.debug("Assigning Data Values from 'data.txt") self.ratio_data = data["Ratio Data"] self.calibration_data = data["Calibration Data"] self.setting_data = data["Setting Data"] #self.setting_data = data["Temperature Alerts"] # conversion_values # schedule_data # light_hour_data # dosage_data = data["Dosage Data"] logger.success("Data Values Updated") return data except (KeyError, ValueError, TypeError): logger.critical("Couldn't Load 'data.txt'") logger.exception("Traceback") logger.info("=" * 125) def load_config(self): try: logger.info("=" * 125) logger.info("Loading config_data") if os.path.isfile('config.json'): with open('config.json', 'r') as json_data_file: config_data = json.loads(json_data_file.read()) logger.success("'config.json' Loaded") logger.debug(f"'config.json' contents: {config_data}") try: logger.info("Assigning Config Values from 'config.json'") self.network_config = config_data["network_config"] self.alert_counter = config_data["alert_counters"] logger.success("Config Values Updated") except (KeyError, ValueError, TypeError): logger.warning("Couldn't Assign Values from 'config.json") except json.JSONDecodeError.with_traceback(): logger.critical("Couldn't Load 'config_data") logger.info("=" * 125) return config_data def get_csv(self): try: with self.lock.acquire(timeout=1): with open('graph_data.csv', 'r') as csv_file: return csv_file.read() except Timeout: logger.warning("Another instance of this application currently holds the lock.") def get_db(self): db = self.cursorObj.execute("SELECT * FROM tank_temperature") return db def sql_connection(self): try: con = sqlite3.connect('AquaPiDB.db') logger.debug("Connection is established to Database") return con except Error: logger.exception(Error) def update(self): g = git.cmd.Git("/home/pi/QTAquarium/") msg = g.pull() logger.info(f"Repo Status: {msg}")
def test_workflow_manager(workflow_start_regular, tmp_path): # For sync between jobs tmp_file = str(tmp_path / "lock") lock = FileLock(tmp_file) lock.acquire() # For sync between jobs flag_file = tmp_path / "flag" flag_file.touch() @ray.remote def long_running(i): lock = FileLock(tmp_file) with lock.acquire(): pass if i % 2 == 0: if flag_file.exists(): raise ValueError() return 100 outputs = [ workflow.run_async(long_running.bind(i), workflow_id=str(i)) for i in range(100) ] # Test list all, it should list all jobs running all_tasks = workflow.list_all() assert len(all_tasks) == 100 all_tasks_running = workflow.list_all(workflow.RUNNING) assert dict(all_tasks) == dict(all_tasks_running) assert workflow.get_status("0") == "RUNNING" # Release lock and make sure all tasks finished lock.release() for o in outputs: try: r = ray.get(o) except Exception: continue assert 100 == r all_tasks_running = workflow.list_all(workflow.WorkflowStatus.RUNNING) assert len(all_tasks_running) == 0 # Half of them failed and half succeed failed_jobs = workflow.list_all("FAILED") assert len(failed_jobs) == 50 finished_jobs = workflow.list_all("SUCCESSFUL") assert len(finished_jobs) == 50 all_tasks_status = workflow.list_all({ workflow.WorkflowStatus.SUCCESSFUL, workflow.WorkflowStatus.FAILED, workflow.WorkflowStatus.RUNNING, }) assert len(all_tasks_status) == 100 assert failed_jobs == [(k, v) for (k, v) in all_tasks_status if v == workflow.WorkflowStatus.FAILED] assert finished_jobs == [(k, v) for (k, v) in all_tasks_status if v == workflow.WorkflowStatus.SUCCESSFUL] # Test get_status assert workflow.get_status("0") == "FAILED" assert workflow.get_status("1") == "SUCCESSFUL" lock.acquire() r = workflow.resume_async("0") assert workflow.get_status("0") == workflow.RUNNING flag_file.unlink() lock.release() assert 100 == ray.get(r) assert workflow.get_status("0") == workflow.SUCCESSFUL # Test cancel lock.acquire() workflow.resume_async("2") assert workflow.get_status("2") == workflow.RUNNING workflow.cancel("2") assert workflow.get_status("2") == workflow.CANCELED # Now resume_all resumed = workflow.resume_all(include_failed=True) assert len(resumed) == 48 lock.release() assert [ray.get(o) for (_, o) in resumed] == [100] * 48
class SimpleCoinbaseBot: def getconf(self, section, key, cast, default): try: val = self.config[section].get(key) except: return default if cast == bool: val = str2bool(val) else: val = cast(val) return val def __init__(self, config): self.cache = {} self.config = config for section, v in CONF_DEFAULTS.items(): for key, cast, default in v: val = self.getconf(section, key, cast, default) if section == 'auth': continue print('SimpleCoinbaseBot config: [{}][{}] -> {}'.format(section, key, val)) setattr(self, key, val) #self.log_file = self.getconf('general', 'log_file', str, 'simplebot.log') #self.cache_file = self.getconf('general', 'cache_file', str, 'simplebot.cache') if not self.cache_file.endswith('.cache'): raise Exception('ERROR: Cache filenames must end in .cache') self.lock_file = self.cache_file.replace('.cache', '.lock') self.lock = FileLock(self.lock_file, timeout=1) try: self.lock.acquire() except: print('ERROR: Failed to acquire lock: {}'.format(self.lock_file)) print('Is another process already running with this config?') exit(1) self.buy_percent_of_wallet = round(self.buy_wallet_percent/100, 4) self.mail_to = self.mail_to.split(',') self.client = self.authenticate() self.wallet = None self.current_price = None self.fee = None self.open_sells = [] self.product_info = None self.min_size = None self.max_size = None self.size_decimal_places = None self.usd_decimal_places = None self.can_buy = False self.current_price_target = None self.current_price_increase = None self.last_buy = None # Run all and validate it worked on init self.get_all() self.__assert() self._open_cache() self.logit('SimpleCoinbaseBot started: {} size-precision:{} usd-precision:{} current-fee:{} min-size:{} max-size:{}'.format( self.coin, self.size_decimal_places, self.usd_decimal_places, self.fee, self.min_size, self.max_size )) self.logit('SimpleCoinbaseBot started: {} sleep_seconds:{} sell_at_percent:{} max_sells_outstanding:{} max_buys_per_hour:{}'.format( self.coin, self.sleep_seconds, self.sell_at_percent, self.max_sells_outstanding, self.max_buys_per_hour )) #self.stoploss_enable = True #self.stoploss_percent = Decimal('-2.0') #self.stoploss_seconds = 86400 def _open_cache(self): if os.path.exists(self.cache_file): with open(self.cache_file, "rb") as f: self.cache = pickle.load(f) def _write_cache(self): with open(self.cache_file+'-tmp', "wb") as f: pickle.dump(self.cache, f) os.fsync(f) if os.path.exists(self.cache_file): os.rename(self.cache_file, self.cache_file+'-prev') os.rename(self.cache_file+'-tmp', self.cache_file) def _log(self, path, msg): now = datetime.now() print('{} {}'.format(now, str(msg).strip())) with open(path, 'a') as f: f.write('{} {}\n'.format(now, str(msg).strip())) def logdebug(self, msg): if self.debug_log_response: self._log(self.debug_log_response_file, msg) def logit(self, msg): if not self.coin in msg: msg = '{} {}'.format(self.coin, msg) self._log(self.log_file, msg) def authenticate(self): key = self.config['auth'].get('key') passphrase = self.config['auth'].get('passphrase') b64secret = self.config['auth'].get('b64secret') auth_client = cbpro.AuthenticatedClient(key, b64secret, passphrase) return auth_client def get_current_price(self): ticker = self.client.get_product_ticker(product_id=self.coin) self.logdebug(ticker) current_price = ticker['price'] return Decimal(current_price) def get_product_info(self): self.product_info = None products = self.client.get_products() for p in products: if p['id'] == self.coin: self.product_info = p break assert(self.product_info != None) self.min_size = Decimal(self.product_info['base_min_size']) self.max_size = Decimal(self.product_info['base_max_size']) # counting the zeros will give the number of decimals to round to self.size_decimal_places = self.product_info['base_increment'].split('1')[0].count('0') self.usd_decimal_places = self.product_info['quote_increment'].split('1')[0].count('0') def get_usd_wallet(self): wallet = None accounts = self.client.get_accounts() for account in accounts: if account['currency'] == 'USD': wallet = account['available'] self.logdebug(account) break return Decimal(wallet) def get_open_sells(self): orders = self.client.get_orders() self.logdebug(orders) open_sells = [] for order in orders: o = order if order['side'] == 'sell' and order['product_id'] == self.coin: order['price'] = Decimal(order['price']) order['size'] = Decimal(order['size']) open_sells.append(order) return open_sells def get_fee(self): """ current pip cbpro version doesn't have my get_fees() patch, so manually query it """ #{'taker_fee_rate': '0.0035', 'maker_fee_rate': '0.0035', 'usd_volume': '21953.58'} fees = self.client._send_message('get', '/fees') assert('taker_fee_rate' in fees) self.logdebug(fees) if Decimal(fees['taker_fee_rate']) > Decimal(fees['maker_fee_rate']): return Decimal(fees['taker_fee_rate']) return Decimal(fees['maker_fee_rate']) def _rand_msleep(self): time.sleep(uniform(0.1, 0.75)) def get_all(self): self._rand_msleep() self.wallet = self.get_usd_wallet() self._rand_msleep() self.get_product_info() self._rand_msleep() self.current_price = self.get_current_price() #self.open_sells = self.get_open_sells() self.fee = self.get_fee() self._rand_msleep() self.get_current_price_target() self._rand_msleep() self.can_buy = self.check_if_can_buy() def sendemail(self, subject, msg=None): """ TODO: Add auth, currently setup to relay locally or relay-by-IP """ for email in self.mail_to: if not email.strip(): continue headers = "From: %s\r\nTo: %s\r\nSubject: %s %s\r\n\r\n" % ( self.mail_from, email, self.coin, subject) if not msg: msg2 = subject else: msg2 = msg msg2 = headers + msg2 server = smtplib.SMTP(self.mail_host) server.sendmail(self.mail_from, email, msg2) server.quit() time.sleep(0.1) def __assert(self): assert(self.wallet != None) assert(self.current_price != None) def maybe_buy_sell(self): self.__assert() if not self.can_buy: return # Check if USD wallet has enough available if self.wallet < Decimal(self.product_info['min_market_funds']): self.logit('WARNING: Wallet value too small (<${}): {}'.format( self.product_info['min_market_funds'], self.wallet)) return # Calculate and check if size is large enough (sometimes it's not if available wallet is too small) buy_amount = round(Decimal(self.buy_percent_of_wallet) * Decimal(self.wallet), self.usd_decimal_places) buy_size = round(Decimal(buy_amount)/self.current_price, self.size_decimal_places) if buy_size <= self.min_size: self.logit('WARNING: Buy size is too small {} < {} wallet:{}.'.format( buy_size, self.min_size, self.wallet)) return # Check if USD wallet has enough available if buy_amount < Decimal(self.product_info['min_market_funds']): self.logit('WARNING: Buy amount too small (<${}): {}'.format( self.product_info['min_market_funds'], buy_amount)) return # Make sure buy_amount is within buy_wallet_min/max if buy_amount < self.buy_wallet_min: self.logit('WARNING: buy_wallet_min hit. Setting to min.') buy_amount = self.buy_wallet_min elif buy_amount > self.buy_wallet_max: self.logit('WARNING: buy_wallet_max hit. Setting to max.') buy_amount = self.buy_wallet_max # adjust size to fit with fee buy_size = round(Decimal(buy_size) - Decimal(buy_size)*Decimal(self.fee), self.size_decimal_places) self.logit('BUY: price:{} amount:{} size:{}'.format( self.current_price, buy_amount, buy_size)) rc = self.client.place_market_order( product_id=self.coin, side='buy', funds=str(buy_amount) ) self.logdebug(rc) self.logit('BUY-RESPONSE: {}'.format(rc)) order_id = rc['id'] errors = 0 self.last_buy = None # Wait until order is completely filled if order_id in self.cache: self.logit('ERROR: order_id exists in cache. ????: {}'.format(order_id)) self.cache[order_id] = { 'first_status':rc, 'last_status':None, 'time':time.time(), 'sell_order':None, 'sell_order_completed':None, 'completed':False, 'profit_usd':None } self._write_cache() done = False error = False status_errors = 0 time.sleep(5) while 1: try: buy = self.client.get_order(order_id) self.cache[order_id]['last_status'] = buy self._write_cache() self.logdebug(buy) if 'settled' in buy: if buy['settled']: self.logit('FILLED: size:{} funds:{}'.format(buy['filled_size'], buy['funds'])) self.last_buy = buy done = True break else: if 'message' in buy: self.logit('WARNING: Failed to get order status: {}'.format(buy['message'])) self.logit('WARNING: Order status failure may be temporary, due to coinbase issues or exchange delays. Check: https://status.pro.coinbase.com') status_errors += 1 else: self.logit('WARNING: Failed to get order status: {}'.format(order_id)) status_errors += 1 time.sleep(10) if status_errors > 10: errors += 1 except Exception as err: self.logit('WARNING: get_order() failed:', err) errors += 1 time.sleep(8) if errors > 5: self.logit('WARNING: Failed to get order. Manual intervention needed.: {}'.format( order_id)) break time.sleep(2) # Buy order done, now place sell if done: rc = self.client.place_limit_order( product_id=self.coin, side='sell', price=str(self.current_price_target), size=str(round(Decimal(self.last_buy['filled_size']), self.size_decimal_places)), ) self.logdebug(rc) self.logit('SELL-RESPONSE: {}'.format(rc)) msg = 'BUY-FILLED: size:{} funds:{}\n'.format(buy['filled_size'], buy['funds']) msg = '{} SELL-PLACED: size:{} price:{}'.format( msg, self.last_buy['filled_size'], self.current_price_target) for m in msg.split('\n'): self.logit(m.strip()) if not self.notify_only_completed: self.sendemail('BUY/SELL', msg=msg) self.cache[order_id]['sell_order'] = rc self._write_cache() self.last_buy = None else: # buy was placed but could not get order status if 'message' in buy: msg = 'BUY-PLACED-NOSTATUS: {}\n'.format(buy['message']) else: msg = 'BUY-PLACED-NOSTATUS: size:{} funds:{}\n'.format( buy['filled_size'], buy['funds']) self.logit(msg) self.sendemail('BUY-ERROR', msg=msg) return buy def run_stoploss(self, buy_order_id): """ Cancel sell order, place new market sell to fill immediately get response and update cache """ v = self.cache[buy_order_id] sell = v['sell_order'] # cancel rc = self.client.cancel_order(sell['id']) self.logdebug(rc) self.logit('STOPLOSS: CANCEL-RESPONSE: {}'.format(rc)) time.sleep(5) # new order rc = self.client.place_market_order( product_id=self.coin, side='sell', size=sell['size'] ) self.logdebug(rc) self.cache[buy_order_id]['sell_order'] = rc self._write_cache() self.logit('STOPLOSS: SELL-RESPONSE: {}'.format(rc)) order_id = rc['id'] time.sleep(5) done = False while 1: try: status = self.client.get_order(order_id) self.logdebug(status) self.cache[buy_order_id]['sell_order'] = status self._write_cache() if 'settled' in status: if status['settled']: self.logit('SELL-FILLED: {}'.format(status)) self.cache[buy_order_id]['sell_order_completed'] = status self._write_cache() done = True break else: if 'message' in status: self.logit('WARNING: Failed to get order status: {}'.format(status['message'])) self.logit('WARNING: Order status failure may be temporary, due to coinbase issues or exchange delays. Check: https://status.pro.coinbase.com') status_errors += 1 else: self.logit('WARNING: Failed to get order status: {}'.format(order_id)) status_errors += 1 time.sleep(10) if status_errors > 10: errors += 1 except Exception as err: self.logit('WARNING: get_order() failed:', err) errors += 1 time.sleep(8) if errors > 5: self.logit('WARNING: Failed to get order. Manual intervention needed.: {}'.format( order_id)) break time.sleep(2) if not done: self.logit('ERROR: Failed to get_order() for stoploss. This is an error and TODO item on how to handle') def check_sell_orders(self): """ Check if any sell orders have completed """ for buy_order_id, v in self.cache.items(): if self.cache[buy_order_id]['completed']: continue if not v['sell_order']: self.logit('WARNING: No sell_order for buy {}. This should not happen.'.format( buy_order_id)) if time.time() - v['time'] > 60*60: self.logit('WARNING: Failed to get order status:') self.logit('WARNING: Writing as done/error since it has been > 30 minutes.') self.cache[buy_order_id]['completed'] = True self._write_cache() continue if 'message' in v['sell_order']: self.logit('WARNING: Corrupted sell order, marking as done: {}'.format(v['sell_order'])) self.cache[buy_order_id]['completed'] = True self.cache[buy_order_id]['sell_order'] = None self._write_cache() continue sell = self.client.get_order(v['sell_order']['id']) if 'message' in sell: self.logit('WARNING: Failed to get sell order status (retrying later): {}'.format( sell['message'])) if time.time() - v['time'] > 60*60: self.logit('WARNING: Failed to get order status:') self.logit('WARNING: Writing as done/error since it has been > 30 minutes.') self.cache[buy_order_id]['completed'] = True self._write_cache() continue if 'status' in sell and sell['status'] != 'open': # calculate profit from buy to sell # done, remove buy/sell self.cache[buy_order_id]['completed'] = True self.cache[buy_order_id]['sell_order_completed'] = sell if sell['status'] == 'done': sell_filled_size = Decimal(sell['filled_size']) sell_value = Decimal(sell['executed_value']) buy_filled_size = Decimal(v['last_status']['filled_size']) buy_value = Decimal(v['last_status']['executed_value']) #buy_sell_diff = round((sell_price*sell_filled_size) - (buy_price*buy_filled_size), 2) buy_sell_diff = round(sell_value - buy_value, 2) done_at = time.mktime(time.strptime(sell['done_at'].split('.')[0], '%Y-%m-%dT%H:%M:%S')) self.cache[buy_order_id]['profit_usd'] = buy_sell_diff msg = 'SELL-COMPLETED: ~duration:{:.2f} bought_val:{} sold_val:{} profit_usd:{}'.format( time.time() - done_at, round(buy_value, 2), round(sell_value, 2), buy_sell_diff ) self.logit(msg) self.sendemail('SELL-COMPLETED', msg=msg) else: self.logit('SELL-COMPLETED-WITH-OTHER-STATUS: {}'.format(sell['status'])) self._write_cache() else: # check for stoploss if enabled if self.stoploss_enable: created_at = time.mktime(time.strptime(sell['created_at'].split('.')[0], '%Y-%m-%dT%H:%M:%S')) duration = time.time() - created_at bought_price = round(Decimal(v['last_status']['executed_value']) / Decimal(v['last_status']['filled_size']), 4) p = 100*(self.current_price/bought_price) - Decimal('100.0') stop_seconds = False stop_percent = False if duration >= self.stoploss_seconds: stop_seconds = True if p <= self.stoploss_percent: stop_percent = True if (stop_seconds or stop_percent) and self.stoploss_strategy == 'report': self.logit('STOPLOSS: percent:{} duration:{}'.format(p, duration)) if self.stoploss_strategy == 'both' and stop_percent and stop_seconds: self.logit('STOPLOSS: running stoploss strategy: {} percent:{} duration:{}'.format( self.stoploss_strategy, p, duration )) self.run_stoploss(buy_order_id) elif self.stoploss_strategy == 'either' and (stop_percent or stop_seconds): self.logit('STOPLOSS: running stoploss strategy: {} percent:{} duration:{}'.format( self.stoploss_strategy, p, duration )) self.run_stoploss(buy_order_id) time.sleep(0.75) def get_current_price_target(self): current_percent_increase = (self.fee*2)+(self.sell_at_percent/100) self.current_price_target = round( self.current_price * current_percent_increase + self.current_price, self.usd_decimal_places ) self.current_price_increase = self.current_price * current_percent_increase return self.current_price_target @property def total_open_orders(self): total = 0 for buy_order_id, v in self.cache.items(): if not v['completed']: total += 1 return total @property def total_sells_in_past_hour(self): current_time = time.time() last_hour_time = current_time - (60*60) total = 0 for buy_order_id, v in self.cache.items(): if v['time'] >= last_hour_time: total += 1 return total def check_if_can_buy(self): """ Check orders if a sell price is <= current_price_target If so, this means no buy is allowed until that order is filled or out of range. Only allow within the fee range though to keep buy/sells further apart. """ self.get_current_price_target() # Check how many buys were placed in past hour and total open if self.total_sells_in_past_hour > self.max_buys_per_hour: self.logit('WARNING: max_buys_per_hour({}) hit'.format(self.max_buys_per_hour)) return # Don't count other orders now, only ones being tracked here #if len(self.open_sells) >= self.max_sells_outstanding: if self.total_open_orders >= self.max_sells_outstanding: self.logit('WARNING: max_sells_outstanding hit ({} of {})'.format( self.total_open_orders, self.max_sells_outstanding)) return False can = True for buy_order_id, v in self.cache.items(): #self.open_sells: if v['completed']: continue sell_order = v['sell_order'] if not sell_order: continue if not 'price' in sell_order: pass #iself.logit('WARNING: Corrupted sell order. Writing as completed (error): {}'.format(sell_order)) #self.cache[buy_order_id]['sell_order'] = None #self.cache[buy_order_id]['completed'] = True #self._write_cache() else: sell_price = Decimal(sell_order['price']) adjusted_sell_price = round(sell_price - (self.fee*2*sell_price), self.usd_decimal_places) if adjusted_sell_price <= self.current_price_target: can = False return can def run(self): # Throttle startups randomly time.sleep(uniform(1, 5)) while 1: self.get_all() self.logit('STATUS: price:{} fee:{} wallet:{} open-sells:{} price-target:{} can-buy:{}'.format( self.current_price, self.fee, self.wallet, self.total_open_orders, self.current_price_target, self.can_buy, )) self.maybe_buy_sell() self.check_sell_orders() time.sleep(self.sleep_seconds)
class ObdIO(object): """Create a obd server""" def __init__(self, port): self.port = port self.ser = None self.__lock = FileLock(gettempdir() + "/yacm-" + self.port.replace("/", "") + ".lock", timeout=1) self.__lock.acquire() def __enter__(self): self.ser = serial.Serial(self.port, parity=serial.PARITY_NONE, stopbits=1, bytesize=8) self.ser.baudrate = 500000 # self.__write("at", "ws") # Reset device self.__write("at", "d0") # Set defaults self.__write("at", "l0") # Disable line feed self.__write("at", "e0") # Disable echo self.__write("at", "h0") # Disable headers self.__write("at", "sp0") # Auto set protocol return self def __exit__(self, exception_type, exception_value, exception_traceback): self.__write("at", "ws") # Reset device self.ser.close() self.__lock.release() os.remove(self.__lock.lock_file) def query(self, mode: str, code: str) -> Union[str, Tuple]: """Query obd requests""" self.__write(mode, code) return self.__read() def __write(self, mode: str, code: str) -> None: self.ser.flushInput() self.ser.write(f"{mode}{code}\r".encode()) logging.info(f"Mode: {mode} Code: {code}") self.ser.flush() if mode == "at": if code == "ws": self.ser.readline() else: self.ser.read_until(b'>') # Discard the "OK" message def __read(self) -> Union[str, Tuple]: raw_data = self.ser.read_until(b'\r>') while raw_data == 0: raw_data = self.ser.read_until(b'\r>') logging.info(f"raw_data: {raw_data}") if raw_data not in {b'\r?\r>', b'?\r\r'}: if raw_data[0] == 13 and raw_data[-3] != 13: # Emulator raw_data = raw_data[1:-2] if raw_data[0] != 13 and raw_data[-3] == 13: # Car raw_data = raw_data[:-3] if raw_data == b"NO DATA": result = "NO DATA" else: result = raw_data.decode("ascii").lower().split(' ')[2:] else: result = "?" logging.info(f"result: {result}\n") return result def supported_pids(self) -> List[str]: """Return supported pids""" hex2bin_map = { "0": "0000", "1": "0001", "2": "0010", "3": "0011", "4": "0100", "5": "0101", "6": "0110", "7": "0111", "8": "1000", "9": "1001", "a": "1010", "b": "1011", "c": "1100", "d": "1101", "e": "1110", "f": "1111", } supported_pids: List[str] = [] for pid in ["00", "20", "40", "60", "80"]: pids = ''.join(self.query("01", pid)) if pids not in {"?", "NO DATA"}: binary_pids = ''.join(hex2bin_map[nibble] for nibble in pids) pid_code = int(pid) for bit in binary_pids: pid_code += 1 if bit == "1": supported_pids.append(hex(pid_code)[2:]) return supported_pids
class CandleConnector(): def __init__(self): self.lock = FileLock("config.csv.lock") # make dict here that stores the amount for each coin self.config = "config.csv" self.candles = candles.BinaceConnector() self.masterTicker = -60 def readConfig(self): self.lock.acquire() df = pd.read_csv(self.config,encoding='utf8', delimiter=',' , names= ['coin', 'capital', 'starting', 'limit', 'currentPrice', 'autobought', 'takeprofit', 'updatetime', 'orderid', 'takeProfitAmount', 'takeProfitOrder', 'delta']) self.lock.release() df.set_index('coin', inplace=True) return df #get the current config def getCoinConfigData(self, coin): df = self.readConfig() return df.loc[coin] #save a new copy of the config def setCoinConfigData(self, df): self.lock.acquire() df.to_csv(f'config.csv', mode='w', header=False, index=True) self.lock.release() def getAutoBoughtAmount(self, coin): return float(self.getCoinConfigData(coin)['autobought']) # helper for buying a number of coins at current price def orderNumber(self, coin, number): return (self.candles.buyMarket(coin, number)) # gives you a quote for a coin def getQuote(self, coin): return float(self.candles.getCoinPrice(coin)) # write out to a log file def logit(self, message, destination): with open(f"testData/{destination}.txt", "a") as f: f.write(message) f.write("\n") def saveCoinBuyData(self, coin, price, amount, setcap=0.0, setupdatetime=180, order="none"): df = self.readConfig() if setcap > 0: df.at[coin, 'capital'] = setcap df.at[coin, 'starting'] = price df.at[coin, 'autobought'] = amount df.at[coin, 'limit'] = price * df.at[coin, 'takeprofit'] df.at[coin, 'updatetime'] = setupdatetime df.at[coin, 'orderid'] = order self.setCoinConfigData(df) def saveCoinLimitData(self, coin, price, limit, setupdatetime=180,): df = self.readConfig() df.at[coin, 'currentPrice'] = price df.at[coin, 'limit'] = limit self.setCoinConfigData(df) def updateDelta(self, coin, delta, price): df = self.readConfig() df.at[coin, 'currentPrice'] = price df.at[coin, 'delta'] = delta -180 self.setCoinConfigData(df) def updateOrder(self, coin, order): df = self.readConfig() df.at[coin, 'orderid'] = order self.setCoinConfigData(df) #sell an amount at current price def sellNow(self, coin): #get the amount the bot bought amount = self.getAutoBoughtAmount(coin) print(f"found {amount}") if amount > 0: print(f"selling") sellorder = self.candles.sellMarket(coin, amount) time.sleep(1) # save the data for analysis later and reset the bot coin's config self.logit(f"SELLING DUE TO TAKEPROFIT {sellorder}", "logger") #need to check to make sure we did sell before we save this sellprice = float(sellorder['fills'][0]['price']) * amount print(sellprice) self.saveCoinBuyData(coin, 0, 0, setcap=sellprice) def echoCurrentTick(self): with open('tick', 'w') as f: f.write(f"{self.masterTicker}") def runForever(self): while 1: self.masterTicker += 60 df = self.readConfig() # loop over the contents of our config file tickers = self.candles.getBook() for coin, row in df.iterrows(): # check to see if the bot has made a purchase position = float(row['autobought']) if position > 0: for x in tickers: if coin == x['symbol']: currentPrice = float(x['bidPrice']) # if the bot has bought, check the update time currentOrder = row['orderid'] updatetime = int(row['updatetime']) delta = self.masterTicker % updatetime if delta== 0: #get the current price and check if it's above our current limit currentLimit = float(row['limit']) if currentPrice < currentLimit: print("checking order") if "none" not in currentOrder: print("cancelOrder") status = self.candles.checkStatus(coin, currentOrder) if status != 'FILLED': self.candles.cancelOrder(coin, currentOrder) time.sleep(.3) print(f"selling because price {currentPrice} < limit {currentLimit}") self.sellNow(coin) else: self.sellNow(coin) else: # calculate a new limit based on our coin's config profile newlimit = currentPrice*float(row['takeprofit']) print(f"new limit {newlimit}") if newlimit > currentLimit: print(f"new limit > {currentLimit}") self.saveCoinLimitData(coin, currentPrice, newlimit) starting = float(row['starting']) # check to see if a stop loss order has been placed if "none" not in currentOrder: print(f"order {currentOrder} is", end = " ") status = self.candles.checkStatus(coin, currentOrder) if status == 'FILLED': print("FILLED so close") sellprice = float(status['fills'][0]['price']) * row['autobought'] self.saveCoinBuyData(coin, 0, 0, setcap=sellprice) print("open") # if stop loss has not been placed, and we are in profit attempt to atleast cover our fees elif currentPrice > starting + (starting * 0.005): print("made our money back placing limit order") #save this order and save to config` order = connector.candles.stopLoss(coin, stop=(starting + (starting * (2 * .0008))), limit=(starting + (starting * (2 * .00076))), position=position)['clientOrderId'] self.updateOrder(coin, order) self.updateDelta(coin, delta, currentPrice) self.logit(f"{self.masterTicker}, {row.starting}, {currentPrice}, {row.limit}", coin) self.echoCurrentTick() time.sleep(60)
class Metric(object): def __init__( self, name: str = None, experiment_id: Optional[str] = None, process_id: int = 0, num_process: int = 1, data_dir: Optional[str] = None, in_memory: bool = False, **kwargs, ): """ A Metrics is the base class and common API for all metrics. Args: process_id (int): specify the id of the node in a distributed settings between 0 and num_nodes-1 This can be used, to compute metrics on distributed setups (in particular non-additive metrics like F1). data_dir (str): path to a directory in which temporary data will be stored. This should be a shared file-system for distributed setups. experiment_id (str): Should be used if you perform several concurrent experiments using the same caching directory (will be indicated in the raise error) in_memory (bool): keep all predictions and references in memory. Not possible in distributed settings. """ # Safety checks assert isinstance( process_id, int ) and process_id >= 0, "'process_id' should be a number greater than 0" assert (isinstance(num_process, int) and num_process > process_id ), "'num_process' should be a number greater than process_id" assert ( process_id == 0 or not in_memory ), "Using 'in_memory' is not possible in distributed setting (process_id > 0)." # Metric name self.name = camelcase_to_snakecase(self.__class__.__name__) # Configuration name self.config_name: str = name or "default" self.process_id = process_id self.num_process = num_process self.in_memory = in_memory self.experiment_id = experiment_id if experiment_id is not None else "cache" self._version = "1.0.0" self._data_dir_root = os.path.expanduser(data_dir or HF_METRICS_CACHE) self.data_dir = self._build_data_dir() # prepare info info = self._info() info.metric_name = self.name info.config_name = self.config_name info.version = self._version self.info = info # Update 'compute' and 'add' docstring self.compute.__func__.__doc__ += self.info.inputs_description self.add_batch.__func__.__doc__ += self.info.inputs_description self.add.__func__.__doc__ += self.info.inputs_description self.arrow_schema = pa.schema(field for field in self.info.features.type) self.buf_writer = None self.writer = None self.writer_batch_size = None self.data = None # Check we can write on the cache file without competitors self.cache_file_name = self._get_cache_path(self.process_id) self.filelock = FileLock(self.cache_file_name + ".lock") try: self.filelock.acquire(timeout=1) except Timeout: raise ValueError( "Cannot acquire lock, caching file might be used by another process, " "you should setup a unique 'experiment_id' for this run.") def _relative_data_dir(self, with_version=True): """Relative path of this dataset in data_dir.""" builder_data_dir = os.path.join(self.name, self.config_name) if not with_version: return builder_data_dir version = self._version version_data_dir = os.path.join(builder_data_dir, str(version)) return version_data_dir def _build_data_dir(self): """ Return the directory for the current version. """ builder_data_dir = os.path.join( self._data_dir_root, self._relative_data_dir(with_version=False)) version_data_dir = os.path.join( self._data_dir_root, self._relative_data_dir(with_version=True)) def _other_versions_on_disk(): """Returns previous versions on disk.""" if not os.path.exists(builder_data_dir): return [] version_dirnames = [] for dir_name in os.listdir(builder_data_dir): try: version_dirnames.append((Version(dir_name), dir_name)) except ValueError: # Invalid version (ex: incomplete data dir) pass version_dirnames.sort(reverse=True) return version_dirnames # Check and warn if other versions exist on disk version_dirs = _other_versions_on_disk() if version_dirs: other_version = version_dirs[0][0] if other_version != self._version: warn_msg = ( "Found a different version {other_version} of metric {name} in " "data_dir {data_dir}. Using currently defined version " "{cur_version}.".format( other_version=str(other_version), name=self.name, data_dir=self._data_dir_root, cur_version=str(self._version), )) logger.warning(warn_msg) os.makedirs(version_data_dir, exist_ok=True) return version_data_dir def _get_cache_path(self, node_id): return os.path.join( self.data_dir, f"{self.experiment_id}-{self.name}-{node_id}.arrow") def finalize(self, timeout=120): """ Close all the writing process and load/gather the data from all the nodes if main node or all_process is True. """ self.writer.finalize() self.writer = None self.buf_writer = None self.filelock.release() if self.process_id == 0: # Let's acquire a lock on each node files to be sure they are finished writing node_files = [] locks = [] for node_id in range(self.num_process): node_file = self._get_cache_path(node_id) filelock = FileLock(node_file + ".lock") filelock.acquire(timeout=timeout) node_files.append({"filename": node_file}) locks.append(filelock) # Read the predictions and references reader = ArrowReader(path=self.data_dir, info=None) self.data = reader.read_files(node_files) # Release all of our locks for lock in locks: lock.release() def compute(self, predictions=None, references=None, timeout=120, **metrics_kwargs): """ Compute the metrics. """ if predictions is not None: self.add_batch(predictions=predictions, references=references) self.finalize(timeout=timeout) self.data.set_format(type=self.info.format) predictions = self.data["predictions"] references = self.data["references"] output = self._compute(predictions=predictions, references=references, **metrics_kwargs) return output def add_batch(self, predictions=None, references=None, **kwargs): """ Add a batch of predictions and references for the metric's stack. """ batch = {"predictions": predictions, "references": references} if self.writer is None: self._init_writer() self.writer.write_batch(batch) def add(self, prediction=None, reference=None, **kwargs): """ Add one prediction and reference for the metric's stack. """ example = {"predictions": prediction, "references": reference} example = self.info.features.encode_example(example) if self.writer is None: self._init_writer() self.writer.write(example) def _init_writer(self): if self.in_memory: self.buf_writer = pa.BufferOutputStream() self.writer = ArrowWriter(schema=self.arrow_schema, stream=self.buf_writer, writer_batch_size=self.writer_batch_size) else: self.buf_writer = None self.writer = ArrowWriter(schema=self.arrow_schema, path=self.cache_file_name, writer_batch_size=self.writer_batch_size) def _info(self) -> MetricInfo: """Construct the MetricInfo object. See `MetricInfo` for details. Warning: This function is only called once and the result is cached for all following .info() calls. Returns: info: (MetricInfo) The metrics information """ raise NotImplementedError def _compute(self, predictions=None, references=None, **kwargs) -> Dict[str, Any]: """ This method defines the common API for all the metrics in the library """ raise NotImplementedError
def __init__(self): lock = FileLock(lock_file) lock.acquire()
from filelock import Timeout, FileLock import sqlite3 import datetime, time while 1: dblock = FileLock("db.lock", timeout=1) dblock.acquire() try: db = sqlite3.connect("./user_gamedata.db3") cursor = db.cursor() # DELETE timeout count = 0 cursor.execute("SELECT * FROM lineupPool") results = cursor.fetchall() for record in results: if round(time.time()) - round(record[1]) > 15: count += 1 cursor.execute("DELETE FROM lineupPool WHERE uid='" + str(record[0]) + "'") db.commit() db.close() # Print message print( str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + ": " + str(count) + " user(s) has/have deleted") except: print("ERROR occured, please check") finally:
yield l[i:i + n] call_dir = Path.cwd() root_dir = Path(sys.path[0]).absolute() queue_dir = Path(queue_dir).absolute() out_dir = Path(out_dir).absolute() done_dir = Path(done_dir).absolute() temp_dir = Path(root_dir / temp_dir).absolute() train_video = False if train_video == "False" else True call_dir, root_dir, queue_dir, out_dir, done_dir, temp_dir, train_video lock = FileLock(root_dir / "gpu.lock") """### Chunk Loop""" with lock.acquire(1): ###### looop #cria uma lista com um snapchot do glob #### if queue_dir is file chunk =[queue] if queue_dir.is_dir(): queue = sorted(list(queue_dir.glob("*"))) else: queue = [queue_dir] for chunk in split_to_batches(queue, queue_batch): print("QUEUE CHUNCK:", chunk) print("net_batch", net_batch, "or len(chunk) if less")
class Metric(MetricInfoMixin): """A Metrics is the base class and common API for all metrics. Args: config_name (``str``): This is used to define a hash specific to a metrics computation script and prevents the metric's data to be overridden when the metric loading script is modified. keep_in_memory (``bool``): keep all predictions and references in memory. Not possible in distributed settings. cache_dir (``str``): Path to a directory in which temporary prediction/references data will be stored. The data directory should be located on a shared file-system in distributed setups. num_process (``int``): specify the total number of nodes in a distributed settings. This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). process_id (``int``): specify the id of the current process in a distributed setup (between 0 and num_process-1) This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). seed (Optional ``int``): If specified, this will temporarily set numpy's random seed when :func:`datasets.Metric.compute` is run. experiment_id (``str``): A specific experiment id. This is used if several distributed evaluations share the same file system. This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). max_concurrent_cache_files (``int``): Max number of concurrent metrics cache files (default 10000). timeout (``Union[int, float]``): Timeout in second for distributed setting synchronization. """ def __init__( self, config_name: Optional[str] = None, keep_in_memory: bool = False, cache_dir: Optional[str] = None, num_process: int = 1, process_id: int = 0, seed: Optional[int] = None, experiment_id: Optional[str] = None, max_concurrent_cache_files: int = 10000, timeout: Union[int, float] = 100, **kwargs, ): # prepare info self.config_name = config_name or "default" info = self._info() info.metric_name = camelcase_to_snakecase(self.__class__.__name__) info.config_name = self.config_name info.experiment_id = experiment_id or "default_experiment" MetricInfoMixin.__init__(self, info) # For easy access on low level # Safety checks on num_process and process_id assert isinstance(process_id, int) and process_id >= 0, "'process_id' should be a number greater than 0" assert ( isinstance(num_process, int) and num_process > process_id ), "'num_process' should be a number greater than process_id" assert ( num_process == 1 or not keep_in_memory ), "Using 'keep_in_memory' is not possible in distributed setting (num_process > 1)." self.num_process = num_process self.process_id = process_id self.max_concurrent_cache_files = max_concurrent_cache_files self.keep_in_memory = keep_in_memory self._data_dir_root = os.path.expanduser(cache_dir or HF_METRICS_CACHE) self.data_dir = self._build_data_dir() self.seed: int = seed or np.random.get_state()[1][0] self.timeout: Union[int, float] = timeout # Update 'compute' and 'add' docstring # methods need to be copied otherwise it changes the docstrings of every instance self.compute = types.MethodType(copyfunc(self.compute), self) self.add_batch = types.MethodType(copyfunc(self.add_batch), self) self.add = types.MethodType(copyfunc(self.add), self) self.compute.__func__.__doc__ += self.info.inputs_description self.add_batch.__func__.__doc__ += self.info.inputs_description self.add.__func__.__doc__ += self.info.inputs_description # self.arrow_schema = pa.schema(field for field in self.info.features.type) self.buf_writer = None self.writer = None self.writer_batch_size = None self.data = None # This is the cache file we store our predictions/references in # Keep it None for now so we can (cloud)pickle the object self.cache_file_name = None self.filelock = None self.rendez_vous_lock = None # This is all the cache files on which we have a lock when we are in a distributed setting self.file_paths = None self.filelocks = None def __len__(self): """Return the number of examples (predictions or predictions/references pair) currently stored in the metric's cache. """ return 0 if self.writer is None else len(self.writer) def __repr__(self): return ( f'Metric(name: "{self.name}", features: {self.features}, ' f'usage: """{self.inputs_description}""", ' f"stored examples: {len(self)})" ) def _build_data_dir(self): """Path of this metric in cache_dir: Will be: self._data_dir_root/self.name/self.config_name/self.hash (if not none)/ If any of these element is missing or if ``with_version=False`` the corresponding subfolders are dropped. """ builder_data_dir = self._data_dir_root builder_data_dir = os.path.join(builder_data_dir, self.name, self.config_name) os.makedirs(builder_data_dir, exist_ok=True) return builder_data_dir def _create_cache_file(self, timeout=1) -> Tuple[str, FileLock]: """ Create a new cache file. If the default cache file is used, we generated a new hash. """ file_path = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{self.process_id}.arrow") filelock = None for i in range(self.max_concurrent_cache_files): filelock = FileLock(file_path + ".lock") try: filelock.acquire(timeout=timeout) except Timeout: # If we have reached the max number of attempts or we are not allow to find a free name (distributed setup) # We raise an error if self.num_process != 1: raise ValueError( f"Error in _create_cache_file: another metric instance is already using the local cache file at {file_path}. " f"Please specify an experiment_id (currently: {self.experiment_id}) to avoid colision " f"between distributed metric instances." ) if i == self.max_concurrent_cache_files - 1: raise ValueError( f"Cannot acquire lock, too many metric instance are operating concurrently on this file system." f"You should set a larger value of max_concurrent_cache_files when creating the metric " f"(current value is {self.max_concurrent_cache_files})." ) # In other cases (allow to find new file name + not yet at max num of attempts) we can try to sample a new hashing name. file_uuid = str(uuid.uuid4()) file_path = os.path.join( self.data_dir, f"{self.experiment_id}-{file_uuid}-{self.num_process}-{self.process_id}.arrow" ) else: break return file_path, filelock def _get_all_cache_files(self) -> Tuple[List[str], List[FileLock]]: """Get a lock on all the cache files in a distributed setup. We wait for timeout second to let all the distributed node finish their tasks (default is 100 seconds). """ if self.num_process == 1: if self.cache_file_name is None: raise ValueError( "Metric cache file doesn't exist. Please make sure that you call `add` or `add_batch` " "at least once before calling `compute`." ) file_paths = [self.cache_file_name] else: file_paths = [ os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{process_id}.arrow") for process_id in range(self.num_process) ] # Let's acquire a lock on each process files to be sure they are finished writing filelocks = [] for process_id, file_path in enumerate(file_paths): filelock = FileLock(file_path + ".lock") try: filelock.acquire(timeout=self.timeout) except Timeout: raise ValueError(f"Cannot acquire lock on cached file {file_path} for process {process_id}.") else: filelocks.append(filelock) return file_paths, filelocks def _check_all_processes_locks(self): expected_lock_file_names = [ os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{process_id}.arrow.lock") for process_id in range(self.num_process) ] for expected_lock_file_name in expected_lock_file_names: nofilelock = FileFreeLock(expected_lock_file_name) try: nofilelock.acquire(timeout=self.timeout) except Timeout: raise ValueError( f"Expected to find locked file {expected_lock_file_name} from process {self.process_id} but it doesn't exist." ) else: nofilelock.release() def _check_rendez_vous(self): expected_lock_file_name = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-0.arrow.lock") nofilelock = FileFreeLock(expected_lock_file_name) try: nofilelock.acquire(timeout=self.timeout) except Timeout: raise ValueError( f"Expected to find locked file {expected_lock_file_name} from process {self.process_id} but it doesn't exist." ) else: nofilelock.release() lock_file_name = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-rdv.lock") rendez_vous_lock = FileLock(lock_file_name) try: rendez_vous_lock.acquire(timeout=self.timeout) except Timeout: raise ValueError(f"Couldn't acquire lock on {lock_file_name} from process {self.process_id}.") else: rendez_vous_lock.release() def _finalize(self): """Close all the writing process and load/gather the data from all the nodes if main node or all_process is True. """ if self.writer is not None: self.writer.finalize() self.writer = None if self.filelock is not None: self.filelock.release() if self.keep_in_memory: # Read the predictions and references reader = ArrowReader(path=self.data_dir, info=DatasetInfo(features=self.features)) self.data = Dataset.from_buffer(self.buf_writer.getvalue()) elif self.process_id == 0: # Let's acquire a lock on each node files to be sure they are finished writing file_paths, filelocks = self._get_all_cache_files() # Read the predictions and references try: reader = ArrowReader(path="", info=DatasetInfo(features=self.features)) self.data = Dataset(**reader.read_files([{"filename": f} for f in file_paths])) except FileNotFoundError: raise ValueError( "Error in finalize: another metric instance is already using the local cache file. " "Please specify an experiment_id to avoid colision between distributed metric instances." ) # Store file paths and locks and we will release/delete them after the computation. self.file_paths = file_paths self.filelocks = filelocks def compute(self, *args, **kwargs) -> Optional[dict]: """Compute the metrics. Args: We disallow the usage of positional arguments to prevent mistakes `predictions` (Optional list/array/tensor): predictions `references` (Optional list/array/tensor): references `**kwargs` (Optional other kwargs): will be forwared to the metrics :func:`_compute` method (see details in the docstring) Return: Dictionnary with the metrics if this metric is run on the main process (process_id == 0) None if the metric is not run on the main process (process_id != 0) """ if args: raise ValueError("Please call `compute` using keyword arguments.") predictions = kwargs.pop("predictions", None) references = kwargs.pop("references", None) if predictions is not None: self.add_batch(predictions=predictions, references=references) self._finalize() self.cache_file_name = None self.filelock = None if self.process_id == 0: self.data.set_format(type=self.info.format) predictions = self.data["predictions"] references = self.data["references"] with temp_seed(self.seed): output = self._compute(predictions=predictions, references=references, **kwargs) if self.buf_writer is not None: self.buf_writer = None del self.data self.data = None else: # Release locks and delete all the cache files for filelock, file_path in zip(self.filelocks, self.file_paths): logger.info(f"Removing {file_path}") del self.data self.data = None del self.writer self.writer = None os.remove(file_path) filelock.release() return output else: return None def add_batch(self, *, predictions=None, references=None): """ Add a batch of predictions and references for the metric's stack. """ batch = {"predictions": predictions, "references": references} batch = self.info.features.encode_batch(batch) if self.writer is None: self._init_writer() try: self.writer.write_batch(batch) except pa.ArrowInvalid: raise ValueError( f"Predictions and/or references don't match the expected format.\n" f"Expected format: {self.features},\n" f"Input predictions: {predictions},\n" f"Input references: {references}" ) def add(self, *, prediction=None, reference=None): """Add one prediction and reference for the metric's stack.""" example = {"predictions": prediction, "references": reference} example = self.info.features.encode_example(example) if self.writer is None: self._init_writer() try: self.writer.write(example) except pa.ArrowInvalid: raise ValueError( f"Prediction and/or reference don't match the expected format.\n" f"Expected format: {self.features},\n" f"Input predictions: {prediction},\n" f"Input references: {reference}" ) def _init_writer(self, timeout=1): if self.num_process > 1: if self.process_id == 0: file_path = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-rdv.lock") self.rendez_vous_lock = FileLock(file_path) try: self.rendez_vous_lock.acquire(timeout=timeout) except TimeoutError: raise ValueError( f"Error in _init_writer: another metric instance is already using the local cache file at {file_path}. " f"Please specify an experiment_id (currently: {self.experiment_id}) to avoid colision " f"between distributed metric instances." ) if self.keep_in_memory: self.buf_writer = pa.BufferOutputStream() self.writer = ArrowWriter( features=self.info.features, stream=self.buf_writer, writer_batch_size=self.writer_batch_size ) else: self.buf_writer = None # Get cache file name and lock it if self.cache_file_name is None or self.filelock is None: cache_file_name, filelock = self._create_cache_file() # get ready self.cache_file_name = cache_file_name self.filelock = filelock self.writer = ArrowWriter( features=self.info.features, path=self.cache_file_name, writer_batch_size=self.writer_batch_size ) # Setup rendez-vous here if if self.num_process > 1: if self.process_id == 0: self._check_all_processes_locks() # wait for everyone to be ready self.rendez_vous_lock.release() # let everyone go else: self._check_rendez_vous() # wait for master to be ready and to let everyone go def _info(self) -> MetricInfo: """Construct the MetricInfo object. See `MetricInfo` for details. Warning: This function is only called once and the result is cached for all following .info() calls. Returns: info: (MetricInfo) The metrics information """ raise NotImplementedError def download_and_prepare( self, download_config: Optional[DownloadConfig] = None, dl_manager: Optional[DownloadManager] = None, **download_and_prepare_kwargs, ): """Downloads and prepares dataset for reading. Args: download_config (Optional ``datasets.DownloadConfig``: specific download configuration parameters. dl_manager (Optional ``datasets.DownloadManager``): specific Download Manger to use """ if dl_manager is None: if download_config is None: download_config = DownloadConfig() download_config.cache_dir = os.path.join(self.data_dir, "downloads") download_config.force_download = False dl_manager = DownloadManager( dataset_name=self.name, download_config=download_config, data_dir=self.data_dir ) self._download_and_prepare(dl_manager) def _download_and_prepare(self, dl_manager): """Downloads and prepares resources for the metric. This is the internal implementation to overwrite called when user calls `download_and_prepare`. It should download all required resources for the metric. Args: dl_manager: (DownloadManager) `DownloadManager` used to download and cache data.. """ return None def _compute(self, *, predictions=None, references=None, **kwargs) -> Dict[str, Any]: """ This method defines the common API for all the metrics in the library """ raise NotImplementedError def __del__(self): if self.filelock is not None: self.filelock.release() if self.rendez_vous_lock is not None: self.rendez_vous_lock.release() del self.writer del self.data
def handle_client(self, conn, addr): """receive request from the client and send response""" self.print_debug("Connection established with client " + addr[0] + ":" + str(addr[1]) + " at " + time.strftime("%a, %d %b %Y %I:%M:%S %p %Z", time.gmtime())) try: while True: _receive_data = [] _tmp = conn.recv(2048) if _tmp: _tmp = _tmp.decode("utf-8") # convert to text string _receive_data.append(_tmp) # append to receive array else: break str_received_data = '' for data in _receive_data: str_received_data += data # convert received array to string print(str_received_data) first_line = str_received_data.split("\r\n", 1) req_type = first_line[0].split(" ")[0] uri = first_line[0].split(" ")[1] headers = {} header_body = str_received_data.split("\r\n\r\n") count = 0 for line in header_body[0].split("\r\n"): if count != 0: headers[line.split(":")[0].strip()] = line.split(":")[1].strip() else: count = count + 1 count = -1 body = '' if len(header_body) > 1: if header_body[0].find("Content-Length:") > -1: last = header_body[0].find("\r\n", header_body[0].find("Content-Length:")) --last count = int(header_body[0][header_body[0].find("Content-Length:") + 16:last]) body = '' for line in header_body[1]: if count > 0: body += line --count http_object = HTTPObject(req_type, uri, headers, body) # Sending code message = 'HTTP/1.1 ' if ".." in uri: self.print_debug("Access Denied " + uri) message += "403 Forbidden" + "\r\n" message += "Content-type: text/plain" + "\r\n" message += "Content-Disposition: inline" + "\r\n\r\n" else: if HTTPObject.get_req_type(http_object).lower() == "get": try: if HTTPObject.get_uri(http_object) == "/": message += "200 OK" + "\r\n" message += "Date: " + formatdate(timeval=None, localtime=False, usegmt=True) + "\r\n" message += "Server:" + socket.gethostname() + "\r\n" self.print_debug("GET DIR " + os.getcwd().replace("\\", "/") + "/" + getattr(self._server_obj, '_path') + HTTPObject.get_uri(http_object)) working_dir = os.getcwd().replace("\\", "/") + "/" + getattr(self._server_obj, '_path') \ + HTTPObject.get_uri(http_object) list_files = os.listdir(working_dir) str_files = '' for file in list_files: str_files += file + "\r\n" message += "Content-Length: " + str(len("{\r\n" + str_files + "}")) + "\r\n" message += "Content-Type: text/directory" + "\r\n" if "Content-Disposition" in HTTPObject.get_headers(http_object): if HTTPObject.get_headers(http_object)["Content-Disposition"] == "attachment": message += "Content-Disposition: attachment/output" + "\r\n" else: message += "Content-Disposition: " + HTTPObject.get_headers(http_object) \ ["Content-Disposition"] + "\r\n" elif "inline" in HTTPObject.get_uri(http_object): message += "Content-Disposition: inline" + "\r\n" else: message += "Content-Disposition: attachment/output" + "\r\n" message += "\r\n" message += "{\r\n" + str_files + "}" else: working_file = os.getcwd().replace("\\", "/") + "/" + getattr(self._server_obj, '_path') \ + HTTPObject.get_uri(http_object) if "Content-Type" in HTTPObject.get_headers(http_object): if HTTPObject.get_headers(http_object)["Content-Type"] == "text/plain": if not working_file.endswith(".txt"): working_file += ".txt" elif HTTPObject.get_headers(http_object)["Content-Type"] == "text/html": if not working_file.endswith(".html"): working_file += ".html" elif HTTPObject.get_headers(http_object)["Content-Type"] == "text/html": if not working_file.endswith(".html"): working_file += ".html" elif HTTPObject.get_headers(http_object)["Content-Type"] == "text/xml": if not working_file.endswith(".xml"): working_file += ".xml" elif HTTPObject.get_headers(http_object)["Content-Type"] == "application/json": if not working_file.endswith(".json"): working_file += ".json" self.print_debug("GET File " + working_file) if not os.path.isfile(working_file): message += "404 Not Found" + "\r\n" message += "Date: " + formatdate(timeval=None, localtime=False, usegmt=True) + "\r\n" message += "Server: " + socket.gethostname() + "\r\n" if "Content-Disposition" in HTTPObject.get_headers(http_object): if HTTPObject.get_headers(http_object)["Content-Disposition"] == "attachment": message += "Content-Disposition: attachment/output" + "\r\n" else: message += "Content-Disposition: " + HTTPObject.get_headers(http_object) \ ["Content-Disposition"] + "\r\n" elif "inline" in HTTPObject.get_uri(http_object): message += "Content-Disposition: inline" + "\r\n" else: message += "Content-Disposition: inline" + "\r\n" message += "\r\n" else: message += "200 OK" + "\r\n" message += "Date: " + formatdate(timeval=None, localtime=False, usegmt=True) + "\r\n" message += "Server: " + socket.gethostname() + "\r\n" fr = open(working_file, 'r') file_data = fr.read() if "Content-Type" in HTTPObject.get_headers(http_object): message += "Content-Type: " + HTTPObject.get_headers(http_object)[ "Content-Type"] + "\r\n" else: if working_file.endswith(".txt"): message += "Content-Type: text/plain" + "\r\n" elif working_file.endswith(".html"): message += "Content-Type: text/html" + "\r\n" elif working_file.endswith(".xml"): message += "Content-Type: text/xml" + "\r\n" elif working_file.endswith(".json"): message += "Content-Type: application/json" + "\r\n" else: message += "Content-Type: text/plain" + "\r\n" message += "Content-Length: " + str(len(file_data)) + "\r\n" if "Content-Disposition" in HTTPObject.get_headers(http_object): if HTTPObject.get_headers(http_object)["Content-Disposition"] == "attachment": message += "Content-Disposition: attachment/output" + "\r\n" else: message += "Content-Disposition: " + HTTPObject.get_headers(http_object) \ ["Content-Disposition"] + "\r\n" elif "inline" in HTTPObject.get_uri(http_object): message += "Content-Disposition: inline" + "\r\n" else: message += "Content-Disposition: inline" + "\r\n" message += "\r\n" message += file_data except OSError as msg: self.print_debug(msg) message = "HTTP/1.1 400 Bad Request\r\n\r\n" message += msg.strerror elif HTTPObject.get_req_type(http_object).lower() == "post": try: if HTTPObject.get_uri(http_object) != "": working_file = os.getcwd().replace("\\", "/") + "/" + getattr(self._server_obj, '_path') \ + HTTPObject.get_uri(http_object) if "Content-Type" in HTTPObject.get_headers(http_object): if HTTPObject.get_headers(http_object)["Content-Type"] == "text/plain": if not working_file.endswith(".txt"): working_file += ".txt" elif HTTPObject.get_headers(http_object)["Content-Type"] == "text/html": if not working_file.endswith(".html"): working_file += ".html" elif HTTPObject.get_headers(http_object)["Content-Type"] == "text/html": if not working_file.endswith(".html"): working_file += ".html" elif HTTPObject.get_headers(http_object)["Content-Type"] == "text/xml": if not working_file.endswith(".xml"): working_file += ".xml" elif HTTPObject.get_headers(http_object)["Content-Type"] == "application/json": if not working_file.endswith(".json"): working_file += ".json" self.print_debug("POST File " + working_file) path = pathlib.Path(working_file) path.parent.mkdir(parents=True, exist_ok=True) lock_path = working_file + ".lock" my_lock = FileLock(lock_path, timeout=2) my_lock.acquire() try: open(working_file, "a").write(HTTPObject.get_data(http_object) + "\n") finally: my_lock.release() message += "200 OK" + "\r\n" message += "Date: " + formatdate(timeval=None, localtime=False, usegmt=True) + "\r\n" message += "Server: " + socket.gethostname() + "\r\n" if "Content-Type" in HTTPObject.get_headers(http_object): message += "Content-Type: " + HTTPObject.get_headers(http_object)[ "Content-Type"] + "\r\n" else: if working_file.endswith(".txt"): message += "Content-Type: text/plain" + "\r\n" elif working_file.endswith(".html"): message += "Content-Type: text/html" + "\r\n" elif working_file.endswith(".json"): message += "Content-Type: application/json" + "\r\n" elif working_file.endswith(".xml"): message += "Content-Type: text/xml" + "\r\n" else: message += "Content-Type: text/plain" + "\r\n" message += "Content-Length: " + str(len(HTTPObject.get_data(http_object))) + "\r\n" if "Content-Disposition" in HTTPObject.get_headers(http_object): if HTTPObject.get_headers(http_object)["Content-Disposition"] == "attachment": message += "Content-Disposition: attachment/output" + "\r\n" else: message += "Content-Disposition: " + HTTPObject.get_headers(http_object) \ ["Content-Disposition"] + "\r\n" elif "inline" in HTTPObject.get_uri(http_object): message += "Content-Disposition: inline" + "\r\n" else: message += "Content-Disposition: inline" + "\r\n" message += "\r\n" message += HTTPObject.get_data(http_object) except OSError as msg: self.print_debug(msg) message = "HTTP/1.1 400 Bad Request\r\n\r\n" message += msg.strerror print("Sending response message to client " + addr[0]) print(message) conn.sendall(bytes(message, 'utf-8')) self.print_debug("Connection closed with client " + addr[0] + ":" + str(addr[1]) + " at " + time.strftime("%a, %d %b %Y %I:%M:%S %p %Z", time.gmtime())) break finally: conn.close()
def main(): parser = argparse.ArgumentParser( description='project management.', formatter_class=argparse.ArgumentDefaultsHelpFormatter, parents=[get_base_parser(tool_version=__version__)]) parser.add_argument('-b', '--base', default="/var/opengrok", help='OpenGrok instance base directory') parser.add_argument('-R', '--roconfig', help='OpenGrok read-only configuration file') parser.add_argument('-U', '--uri', default='http://localhost:8080/source', help='URI of the webapp with context path') parser.add_argument('-c', '--configmerge', help='path to the ConfigMerge binary') parser.add_argument('--java', help='Path to java binary ' '(needed for config merge program)') parser.add_argument('-j', '--jar', help='Path to jar archive to run') parser.add_argument('-u', '--upload', action='store_true', help='Upload configuration at the end') parser.add_argument('-n', '--noop', action='store_true', default=False, help='Do not run any commands or modify any config' ', just report. Usually implies ' 'the --debug option.') parser.add_argument('-N', '--nosourcedelete', action='store_true', default=False, help='Do not delete source code when ' 'deleting a project') add_http_headers(parser) parser.add_argument( '--api_timeout', type=int, help='Set response timeout in seconds for RESTful API calls') group = parser.add_mutually_exclusive_group() group.add_argument('-a', '--add', metavar='project', nargs='+', help='Add project (assumes its source is available ' 'under source root') group.add_argument('-d', '--delete', metavar='project', nargs='+', help='Delete project and its data and source code') group.add_argument('-r', '--refresh', action='store_true', help='Refresh configuration. If read-only ' 'configuration is supplied, it is merged ' 'with current ' 'configuration.') try: args = parser.parse_args() except ValueError as e: fatal(e) doit = not args.noop configmerge = None # # Setup logger as a first thing after parsing arguments so that it can be # used through the rest of the program. # logger = get_console_logger(get_class_basename(), args.loglevel) headers = get_headers(args.header) if args.nosourcedelete and not args.delete: logger.error("The no source delete option is only valid for delete") sys.exit(FAILURE_EXITVAL) # Set the base directory if args.base: if path.isdir(args.base): logger.debug("Using {} as instance base".format(args.base)) else: logger.error( "Not a directory: {}\n" "Set the base directory with the --base option.".format( args.base)) sys.exit(FAILURE_EXITVAL) # If read-only configuration file is specified, this means read-only # configuration will need to be merged with active webapp configuration. # This requires config merge tool to be run so couple of other things # need to be checked. if args.roconfig: if path.isfile(args.roconfig): logger.debug("Using {} as read-only config".format(args.roconfig)) else: logger.error("File {} does not exist".format(args.roconfig)) sys.exit(FAILURE_EXITVAL) configmerge_file = get_command(logger, args.configmerge, "opengrok-config-merge") if configmerge_file is None: logger.error("Use the --configmerge option to specify the path to" "the config merge script") sys.exit(FAILURE_EXITVAL) configmerge = [configmerge_file] if args.loglevel: configmerge.append('-l') configmerge.append(str(args.loglevel)) if args.jar is None: logger.error('jar file needed for config merge tool, ' 'use --jar to specify one') sys.exit(FAILURE_EXITVAL) uri = args.uri if not is_web_uri(uri): logger.error("Not a URI: {}".format(uri)) sys.exit(FAILURE_EXITVAL) logger.debug("web application URI = {}".format(uri)) lock = FileLock( path.join(tempfile.gettempdir(), path.basename(sys.argv[0]) + ".lock")) try: with lock.acquire(timeout=0): if args.add: for proj in args.add: project_add(doit=doit, logger=logger, project=proj, uri=uri, headers=headers, timeout=args.api_timeout) config_refresh(doit=doit, logger=logger, basedir=args.base, uri=uri, configmerge=configmerge, jar_file=args.jar, roconfig=args.roconfig, java=args.java, headers=headers, timeout=args.api_timeout) elif args.delete: for proj in args.delete: project_delete(logger=logger, project=proj, uri=uri, doit=doit, deletesource=not args.nosourcedelete, headers=headers, timeout=args.api_timeout) config_refresh(doit=doit, logger=logger, basedir=args.base, uri=uri, configmerge=configmerge, jar_file=args.jar, roconfig=args.roconfig, java=args.java, headers=headers, timeout=args.api_timeout) elif args.refresh: config_refresh(doit=doit, logger=logger, basedir=args.base, uri=uri, configmerge=configmerge, jar_file=args.jar, roconfig=args.roconfig, java=args.java, headers=headers, timeout=args.api_timeout) else: parser.print_help() sys.exit(FAILURE_EXITVAL) if args.upload: main_config = get_config_file(basedir=args.base) if path.isfile(main_config): if doit: with io.open(main_config, mode='r', encoding="utf-8") as config_file: config_data = config_file.read().encode("utf-8") if not set_configuration(logger, config_data, uri, headers=headers, timeout=args.api_timeout): sys.exit(FAILURE_EXITVAL) else: logger.error("file {} does not exist".format(main_config)) sys.exit(FAILURE_EXITVAL) except Timeout: logger.warning("Already running, exiting.") sys.exit(FAILURE_EXITVAL)
class Db(object): """ Basic file storage for a JSON object. """ def where_not_in(self, column, *values): """ Selects results which do not match the given column/values expression. Args: column (str): The named field to test against. values (str): Vales to search for. A record will not be returned if the field named in *column* is contained inside of the list of values given. """ return [x for x in self.data if x[column] not in values] def all_jobs(self): "Retrieve all records." return self.data def values(self): "Synonym for #all_jobs." return self.data def find_by_remote_job_id(self, job_id): """ Finds a record by the id number for that job on the remote cluster. Args: job_id(str): the job id for this job as scheduled or running on the remote. Returns: A :class:`dict` object containing various attributes of the job on the local and the remote:: { local_id: '123123', local_state: 3, local_wd: '/var/sge_working_dir', remote_id: '1234324', remote_state: 2, remote_wd: '/var/remote_working_dir', last_checked: '2017-12-27 16:35:30.898984' } """ return next((x for x in self.values() if x[R_ID] == job_id), None) def find_by_local_job_id(self, job_id): return next((x for x in self.values() if x[L_ID] == job_id), None) def insert(self, local_id, local_state, local_wd, remote_id, remote_state, remote_wd): self.data.append( dict(local_id=local_id, local_state=local_state, local_wd=local_wd, remote_id=remote_id, remote_state=remote_state, remote_wd=remote_wd, last_checked=str(datetime.now()))) def update(self, job): job['last_checked'] = str(datetime.now()) def delete(self, job): return self.data.remove(job) def __init__(self): self.lock = FileLock(LOCKFILE) self.data = None def save(self): with open(DBFILE, 'w') as file: file.write(dumps(self.data)) def open(self): self.lock.acquire(timeout=2) with open(DBFILE) as file: self.data = loads(file.read()) def close(self): self.save() self.data = None self.lock.release() def __enter__(self): self.open() return self def __exit__(self, x, y, z): "Exit method for resource" self.close()
def _instance_iterator(self, file_path: str) -> Iterable[Instance]: cache_file: Optional[str] = None if self._cache_directory: cache_file = self._get_cache_location_for_file_path(file_path) if cache_file is not None and os.path.exists(cache_file): cache_file_lock = FileLock(cache_file + ".lock", timeout=self.CACHE_FILE_LOCK_TIMEOUT) try: cache_file_lock.acquire() # We make an assumption here that if we can obtain the lock, no one will # be trying to write to the file anymore, so it should be safe to release the lock # before reading so that other processes can also read from it. cache_file_lock.release() logger.info("Reading instances from cache %s", cache_file) with open(cache_file) as data_file: yield from self._multi_worker_islice( data_file, transform=self.deserialize_instance) except Timeout: logger.warning( "Failed to acquire lock on dataset cache file within %d seconds. " "Cannot use cache to read instances.", self.CACHE_FILE_LOCK_TIMEOUT, ) yield from self._multi_worker_islice(self._read(file_path), ensure_lazy=True) elif cache_file is not None and not os.path.exists(cache_file): instances = self._multi_worker_islice(self._read(file_path), ensure_lazy=True) # The cache file doesn't exist so we'll try writing to it. if self.max_instances is not None: # But we don't write to the cache when max_instances is specified. logger.warning( "Skipping writing to data cache since max_instances was specified." ) yield from instances elif util.is_distributed() or (get_worker_info() and get_worker_info().num_workers): # We also shouldn't write to the cache if there's more than one process loading # instances since each worker only receives a partial share of the instances. logger.warning( "Can't cache data instances when there are multiple processes loading data" ) yield from instances else: try: with FileLock(cache_file + ".lock", timeout=self.CACHE_FILE_LOCK_TIMEOUT): with CacheFile(cache_file, mode="w+") as cache_handle: logger.info("Caching instances to temp file %s", cache_handle.name) for instance in instances: cache_handle.write( self.serialize_instance(instance) + "\n") yield instance except Timeout: logger.warning( "Failed to acquire lock on dataset cache file within %d seconds. " "Cannot write to cache.", self.CACHE_FILE_LOCK_TIMEOUT, ) yield from instances else: # No cache. yield from self._multi_worker_islice(self._read(file_path), ensure_lazy=True)
def main(): parser = argparse.ArgumentParser( description='Manage parallel workers.', formatter_class=argparse.ArgumentDefaultsHelpFormatter, parents=[get_base_parser(tool_version=__version__)]) parser.add_argument('-w', '--workers', default=multiprocessing.cpu_count(), help='Number of worker processes', type=int) # There can be only one way how to supply list of projects to process. group1 = parser.add_mutually_exclusive_group() group1.add_argument('-d', '--directory', help='Directory to process') group1.add_argument('-P', '--project', nargs='*', help='project(s) to process') parser.add_argument('-I', '--indexed', action='store_true', help='Sync indexed projects only') parser.add_argument('-i', '--ignore_errors', nargs='*', help='ignore errors from these projects') parser.add_argument('--ignore_project', nargs='+', help='do not process given project(s)') parser.add_argument('-c', '--config', required=True, help='config file in JSON/YAML format') parser.add_argument('-U', '--uri', default='http://localhost:8080/source', help='URI of the webapp with context path') parser.add_argument('-f', '--driveon', action='store_true', default=False, help='continue command sequence processing even ' 'if one of the commands requests break') parser.add_argument('--nolock', action='store_false', default=True, help='do not acquire lock that prevents multiple ' 'instances from running') parser.add_argument('--api_timeout', type=int, default=3, help='Set response timeout in seconds' 'for RESTful API calls') parser.add_argument( '--async_api_timeout', type=int, default=300, help='Set timeout in seconds for asynchronous REST API calls') parser.add_argument('--check_config', action='store_true', help='check configuration and exit') add_http_headers(parser) try: args = parser.parse_args() except ValueError as e: return fatal(e, exit=False) logger = get_console_logger(get_class_basename(), args.loglevel) uri = args.uri if not is_web_uri(uri): logger.error("Not a URI: {}".format(uri)) return FAILURE_EXITVAL logger.debug("web application URI = {}".format(uri)) # First read and validate configuration file as it is mandatory argument. config = read_config(logger, args.config) if config is None: logger.error("Cannot read config file from {}".format(args.config)) return FAILURE_EXITVAL # Changing working directory to root will avoid problems when running # programs via sudo/su. Do this only after the config file was read # so that its path can be specified as relative. try: os.chdir("/") except OSError: logger.error("cannot change working directory to /", exc_info=True) return FAILURE_EXITVAL try: commands = config["commands"] except KeyError: logger.error("The config file has to contain key \"commands\"") return FAILURE_EXITVAL headers = get_headers(args.header) config_headers = config.get("headers") if config_headers: logger.debug( "Updating HTTP headers with headers from the configuration: {}". format(config_headers)) headers.update(config_headers) directory = args.directory if not args.directory and not args.project and not args.indexed: # Assume directory, get the source root value from the webapp. directory = get_config_value(logger, 'sourceRoot', uri, headers=headers, timeout=args.api_timeout) if not directory: logger.error("Neither -d or -P or -I specified and cannot get " "source root from the webapp") return FAILURE_EXITVAL else: logger.info("Assuming directory: {}".format(directory)) ignore_errors = [] if args.ignore_errors: ignore_errors = args.ignore_errors else: try: ignore_errors = config["ignore_errors"] except KeyError: pass logger.debug("Ignoring errors from projects: {}".format(ignore_errors)) dirs_to_process = [] if args.project: dirs_to_process = args.project logger.debug("Processing directories: {}".format(dirs_to_process)) elif args.indexed: indexed_projects = list_indexed_projects(logger, uri, headers=headers, timeout=args.api_timeout) logger.debug( "Processing indexed projects: {}".format(indexed_projects)) if indexed_projects: for line in indexed_projects: dirs_to_process.append(line.strip()) else: logger.error("cannot get list of projects") return FAILURE_EXITVAL else: logger.debug("Processing directory {}".format(directory)) for entry in os.listdir(directory): if path.isdir(path.join(directory, entry)): dirs_to_process.append(entry) ignored_projects = [] config_ignored_projects = config.get("ignore_projects") if config_ignored_projects: logger.debug( "Updating list of ignored projects list from the configuration: {}" .format(config_ignored_projects)) ignored_projects.extend(config_ignored_projects) if args.ignore_project: logger.debug( "Updating list of ignored projects based on options: {}".format( args.ignore_project)) ignored_projects.extend(args.ignore_project) if ignored_projects: dirs_to_process = list(set(dirs_to_process) - set(ignored_projects)) logger.debug("Removing projects: {}".format(ignored_projects)) logger.debug("directories to process: {}".format(dirs_to_process)) if args.project and len(args.project) == 1: lockfile_name = args.project[0] else: lockfile_name = os.path.basename(sys.argv[0]) if args.nolock: try: r = do_sync(args.loglevel, commands, config.get("cleanup"), dirs_to_process, ignore_errors, uri, args.workers, driveon=args.driveon, http_headers=headers, timeout=args.api_timeout, api_timeout=args.async_api_timeout, check_config=args.check_config) except CommandConfigurationException as exc: logger.error("Invalid configuration: {}".format(exc)) return FAILURE_EXITVAL else: lock = FileLock( os.path.join(tempfile.gettempdir(), lockfile_name + ".lock")) try: with lock.acquire(timeout=0): try: r = do_sync(args.loglevel, commands, config.get("cleanup"), dirs_to_process, ignore_errors, uri, args.workers, driveon=args.driveon, http_headers=headers, timeout=args.api_timeout, api_timeout=args.async_api_timeout, check_config=args.check_config) except CommandConfigurationException as exc: logger.error("Invalid configuration: {}".format(exc)) return FAILURE_EXITVAL except Timeout: logger.warning("Already running") return FAILURE_EXITVAL return r
class CandleConnector(): def __init__(self): self.lock = FileLock("config.csv.lock") # make dict here that stores the amount for each coin self.config = "config.csv" self.candles = candles.BinaceConnector() def readConfig(self): self.lock.acquire() df = pd.read_csv(self.config, encoding='utf8', delimiter=',', names=[ 'coin', 'capital', 'starting', 'limit', 'currentPrice', 'autobought', 'takeprofit', 'updatetime', 'orderid', 'takeProfitAmount', 'takeProfitOrder', 'delta' ]) self.lock.release() df.set_index('coin', inplace=True) return df #get the current config def getCoinConfigData(self, coin): df = self.readConfig() return df.loc[coin] #save a new copy of the config def setCoinConfigData(self, df): self.lock.acquire() df.to_csv(f'config.csv', mode='w', header=False, index=True) self.lock.release() def getAutoBoughtAmount(self, coin): return float(self.getCoinConfigData(coin)['autobought']) # helper for buying a number of coins at current price def orderNumber(self, coin, number): return (self.candles.buyMarket(coin, number)) # gives you a quote for a coin def getQuote(self, coin): return float(self.candles.getCoinPrice(coin)) def getBuyPower(self): return float(self.candles.getUSD()) # set an order for an number amount def orderAmount(self, coin, amount): return (self.candles.order_buy_crypto_by_price(coin, amount)) # write out to a log file def logit(self, message, destination): with open(f"testData/{destination}.txt", "a") as f: f.write(message) f.write("\n") def saveCoinBuyData(self, coin, price, amount, setcap=None, setupdatetime=180, order="none"): df = self.readConfig() if setcap is not None: df.at[coin, 'capital'] = setcap df.at[coin, 'starting'] = price df.at[coin, 'autobought'] = amount df.at[coin, 'limit'] = price * df.at[coin, 'takeprofit'] df.at[coin, 'updatetime'] = setupdatetime df.at[coin, 'orderid'] = order self.setCoinConfigData(df) def saveCoinLimitBuyData(self, coin, price, amount, order): df = self.readConfig() df.at[coin, 'starting'] = price df.at[coin, 'takeProfitAmount'] = amount df.at[coin, 'takeProfitOrder'] = order self.setCoinConfigData(df) # check to see how much can be purchased with the current capital # then purchase that amount of coins def buyForProfit(self, coin, strat=None): coinsCapital = self.getCoinConfigData(coin)['capital'] avalFunds = self.getBuyPower() print( f"capital {coinsCapital} above funds {avalFunds} is {coinsCapital > avalFunds}" ) if (coinsCapital > avalFunds): return 0 if float(self.getCoinConfigData(coin)['autobought']) > 0: return 0 price = self.getQuote(coin) #TODO add logic that allows for multiple strategies that will #allow for different allocations of the starting capital bought = float(coinsCapital / self.getQuote(coin)) minOrder = None minNot = None print(bought) #grab the trading rules for the coin for filt in (self.candles.getCoinInfo(coin)['filters']): if filt['filterType'] == "LOT_SIZE": minOrder = float(filt['minQty']) if filt['filterType'] == 'MIN_NOTIONAL': minNot = float(filt['minNotional']) mod = bought % minOrder #make sure the amount we are buying is standardized for Binance if mod: bought = bought - mod #this needs to get the perciesion from the filter bought = round(bought, int(self.candles.getCoinInfo(coin)['quotePrecision'])) print(bought) if (bought * price) > minNot: order = self.orderNumber(coin, bought) self.saveCoinBuyData(coin, price, bought) self.logit(f"BUYING {order}", "logger") #reset our coin data so we can have a current graph file = pathlib.Path(f"testData/{coin}.txt") if file.exists(): os.rename(f"testData/{coin}.txt", f"testData/{coin}{datetime.now()}.txt") else: bought = None self.logit( f"Failed to buy {bought}, {coin}. Due minNotional of {minNot}", "logger") return bought, price # check to see how much can be purchased with the current capital # then purchase that amount of coins def buyForLimit(self, coin, strat=None): #TODO seperate the capital out so we can run both at the same time coinsCapital = float(self.getCoinConfigData(coin)['capital']) avalFunds = self.getBuyPower() prevoiusID = self.getCoinConfigData(coin)['takeProfitOrder'] print(avalFunds) print(coinsCapital) if (coinsCapital > avalFunds): print("no money") return None #test here to see if previous order has been filled previousOrder = self.getCoinConfigData(coin)['takeProfitOrder'] print(f"Prevoius {previousOrder}") if previousOrder != "none": print("previous order") status = self.candles.checkStatus(coin, previousOrder) #if previous status has been filled. We need to save the winnings to new capital print(f"status {status}") if 'FILLED' != status: print("previous has not filled yet") return None print("previous has been filled so let's buy more") price = self.getQuote(coin) #TODO add logic that allows for multiple strategies that will #allow for different allocations of the starting capital bought = float(coinsCapital / self.getQuote(coin)) minOrder = None minNot = None print(bought) #grab the trading rules for the coin for filt in (self.candles.getCoinInfo(coin)['filters']): if filt['filterType'] == "LOT_SIZE": minOrder = float(filt['minQty']) if filt['filterType'] == 'MIN_NOTIONAL': minNot = float(filt['minNotional']) mod = bought % minOrder #make sure the amount we are buying is standardized for Binance if mod: bought = bought - mod #this needs to get the perciesion from the filter bought = round(bought, int(self.candles.getCoinInfo(coin)['quotePrecision'])) print(bought) if (bought * price) > minNot: order = self.orderNumber(coin, bought) self.logit(f"BUYING {order}", "logger") #reset our coin data so we can have a current graph file = pathlib.Path(f"testData/{coin}.txt") if file.exists(): os.rename(f"testData/{coin}.txt", f"testData/{coin}{datetime.now()}.txt") return order else: bought = None self.logit( f"Failed to buy {bought}, {coin}. Due minNotional of {minNot}", "logger") return None #sell an amount at current price def sellNow(self, coin): #get the amount the bot bought amount = self.getAutoBoughtAmount(coin) print(f"we have {amount}") if amount > 0: print(f"selling") #NEED TO CHECK TO SEE IF WE CURRNTLY HAVE AN ORDER orderID = self.getCoinConfigData(coin)['orderid'] print(f"previous order {orderID}") if "none" not in orderid: print("cancelOrder") self.candles.cancelOrder(coin, orderid) time.sleep(.3) # self.candles.testOrder(coin, SIDE_SELL, amount) sellorder = self.candles.sellMarket(coin, amount) orderID = sellorder['clientOrderId'] # save the data for analysis later and reset the bot coin's config self.logit(f"SELLING DUE TO STRAT {sellorder}", "logger") sellprice = float(sellorder['fills'][0]['price']) * amount print(sellprice) self.saveCoinBuyData(coin, 0, 0, setcap=sellprice) def doMaxProfit(self, coin, action): self.logit(f"buysell {action}", "logger") self.logit(f"symbol {coin}", "logger") if action == 'sell': self.sellNow(coin) if action == 'buy': self.buyForProfit(coin) def doTakeProfit(self, coin, action): self.logit(f"buysell limit {action}", "logger") self.logit(f"symbol limit {coin}", "logger") #add sell logic if action == 'buy': order = self.buyForLimit(coin) time.sleep(.5) if order is not None: price = float(order['fills'][0]['price']) limit = price * 1.01 amount = float(order['fills'][0]['qty']) limit_order = self.candles.sellLimit(coin, amount, limit) self.saveCoinLimitBuyData(coin, price, limit, limit_order['clientOrderId']) elif action == 'sell': previousOrder = self.getCoinConfigData(coin)['takeProfitOrder'] if previousOrder != "none": print("previous order") status = self.candles.checkStatus(coin, previousOrder) #if previous status has been filled. We need to save the winnings to new capital print(f"status {status}") if 'FILLED' != status: print("previous has not filled yet so we gonna cancel") self.candles.cancelOrder(coin, previousOrder) time.sleep(.3) self.sellNow(coin) return None print("previous has been filled so nothing to do")
def main(): ret = SUCCESS_EXITVAL parser = argparse.ArgumentParser( description='project mirroring', formatter_class=argparse.ArgumentDefaultsHelpFormatter, parents=[get_base_parser(tool_version=__version__)]) parser.add_argument('project', nargs='*', default=None) parser.add_argument('-a', '--all', action='store_true', help='mirror all indexed projects', default=False) parser.add_argument('-c', '--config', help='config file in JSON/YAML format') parser.add_argument('--check_config', action='store_true', help='check configuration and exit') parser.add_argument('-U', '--uri', default='http://localhost:8080/source', help='uri of the webapp with context path') parser.add_argument('-b', '--batch', action='store_true', help='batch mode - will log into a file') parser.add_argument('-L', '--logdir', help='log directory') parser.add_argument('-B', '--backupcount', default=8, type=int, help='how many log files to keep around in batch mode') parser.add_argument('-I', '--check-changes', action='store_true', help='Check for changes in the project or its' ' repositories,' ' terminate the processing' ' if no change is found.') parser.add_argument( '--strip-outgoing', action='store_true', default=False, help='check outgoing changes for each repository of a project,' 'strip any such changes and remove all project data so that' 'it can be reindexed from scratch. Supported: Git') parser.add_argument('-w', '--workers', default=cpu_count(), type=int, help='Number of worker processes') add_http_headers(parser) parser.add_argument('--api_timeout', type=int, default=3, help='Set response timeout in seconds ' 'for RESTful API calls') parser.add_argument( '--async_api_timeout', type=int, default=300, help='Set timeout in seconds for asynchronous REST API calls') try: args = parser.parse_args() except ValueError as e: return fatal(e, False) logger = get_console_logger(get_class_basename(), args.loglevel) if args.config: config = read_config(logger, args.config) if config is None: return fatal("Cannot read config file from {}".format(args.config), False) else: config = {} if not check_configuration(config): logger.error("Configuration check failed, exiting") return 1 if args.check_config: logger.info("Configuration check passed, exiting") return 0 nomirror = os.environ.get(OPENGROK_NO_MIRROR_ENV) if nomirror and len(nomirror) > 0: logger.debug( "skipping mirror based on the {} environment variable".format( OPENGROK_NO_MIRROR_ENV)) return SUCCESS_EXITVAL if len(args.project) > 0 and args.all: return fatal("Cannot use both project list and -a/--all", False) if not args.all and len(args.project) == 0: return fatal("Need at least one project or --all", False) uri = args.uri if not is_web_uri(uri): return fatal("Not a URI: {}".format(uri), False) logger.debug("web application URI = {}".format(uri)) headers = get_headers(args.header) # Save the source root to avoid querying the web application. source_root = get_config_value(logger, 'sourceRoot', uri, headers=headers, timeout=args.api_timeout) if not source_root: return 1 logger.debug("Source root = {}".format(source_root)) hookdir = config.get(HOOKDIR_PROPERTY) if hookdir: logger.debug("Hook directory = {}".format(hookdir)) command_timeout = get_int(logger, "command timeout", config.get(CMD_TIMEOUT_PROPERTY)) if command_timeout: logger.debug("Global command timeout = {}".format(command_timeout)) hook_timeout = get_int(logger, "hook timeout", config.get(HOOK_TIMEOUT_PROPERTY)) if hook_timeout: logger.debug("Global hook timeout = {}".format(hook_timeout)) logdir = None # Log messages to dedicated log file if running in batch mode. if args.batch: if args.logdir: logdir = args.logdir else: logdir = config.get(LOGDIR_PROPERTY) if not logdir: return fatal( "The {} property is required in batch mode".format( LOGDIR_PROPERTY), False) projects = args.project if len(projects) == 1: lockfile = projects[0] + "-mirror" else: lockfile = os.path.basename(sys.argv[0]) if args.all: projects = list_indexed_projects(logger, args.uri, headers=headers, timeout=args.api_timeout) lock = FileLock(os.path.join(tempfile.gettempdir(), lockfile + ".lock")) try: with lock.acquire(timeout=0): with Pool(processes=int(args.workers)) as pool: worker_args = [] for x in projects: worker_args.append([ x, logdir, args.loglevel, args.backupcount, config, args.check_changes, args.strip_outgoing, args.uri, source_root, args.batch, headers, args.api_timeout, args.async_api_timeout ]) try: project_results = pool.map(worker, worker_args, 1) except KeyboardInterrupt: return FAILURE_EXITVAL else: if any([x == FAILURE_EXITVAL for x in project_results]): ret = FAILURE_EXITVAL if all([x == CONTINUE_EXITVAL for x in project_results]): ret = CONTINUE_EXITVAL except Timeout: logger.warning("Already running, exiting.") return FAILURE_EXITVAL logging.shutdown() return ret
def test_wf_in_actor(workflow_start_regular, tmp_path): fail_flag = tmp_path / "fail" cnt = tmp_path / "count" cnt.write_text(str(0)) lock_file = tmp_path / "lock" @workflow.step def start_session(): if fail_flag.exists(): raise Exception() v = int(cnt.read_text()) + 1 cnt.write_text(str(v)) with FileLock(str(lock_file)): return "UP" @workflow.virtual_actor class Session: def __init__(self): self._session_status = "DOWN" @workflow.virtual_actor.readonly def get_status(self): return self._session_status def update_session(self, up): (ret, err) = up if err is None: self._session_status = ret else: self._session_status = err return self._session_status def session_start(self): step = start_session.step() return step def session_start_with_status(self): self._session_status = "STARTING" return self.update_session.step( start_session.options(catch_exceptions=True).step()) def __getstate__(self): return self._session_status def __setstate__(self, state): self._session_status = state actor = Session.get_or_create("session_id") fail_flag.touch() with pytest.raises(Exception): actor.session_start.run() fail_flag.unlink() ray.get(workflow.resume("session_id")) # After resume, it'll rerun start_session which will # generate 1 assert cnt.read_text() == "1" assert actor.session_start.run() == "UP" assert cnt.read_text() == "2" assert actor.session_start_with_status.run() == "UP" assert cnt.read_text() == "3" # Now test a new session. actor = Session.get_or_create("session_id") fail_flag.touch() assert isinstance(actor.session_start_with_status.run(), Exception) assert cnt.read_text() == "3" lock = FileLock(str(lock_file)) lock.acquire() fail_flag.unlink() ret = actor.session_start_with_status.run_async() for i in range(0, 60): if cnt.read_text() == "4": break time.sleep(1) assert cnt.read_text() == "4" # This means when return from session_start_with_status, # the session got updated assert actor.get_status.run() == "STARTING" lock.release() assert ray.get(ret) == "UP"
def main(): dirs_to_process = [] parser = argparse.ArgumentParser( description='Manage parallel workers.', parents=[get_baseparser(tool_version=__version__)]) parser.add_argument('-w', '--workers', default=multiprocessing.cpu_count(), help='Number of worker processes') # There can be only one way how to supply list of projects to process. group1 = parser.add_mutually_exclusive_group() group1.add_argument('-d', '--directory', help='Directory to process') group1.add_argument('-P', '--projects', nargs='*', help='List of projects to process') parser.add_argument('-I', '--indexed', action='store_true', help='Sync indexed projects only') parser.add_argument('-i', '--ignore_errors', nargs='*', help='ignore errors from these projects') parser.add_argument('-c', '--config', required=True, help='config file in JSON/YAML format') parser.add_argument('-U', '--uri', default='http://localhost:8080/source', help='URI of the webapp with context path') parser.add_argument('-f', '--driveon', action='store_true', default=False, help='continue command sequence processing even ' 'if one of the commands requests break') try: args = parser.parse_args() except ValueError as e: fatal(e) logger = get_console_logger(get_class_basename(), args.loglevel) uri = args.uri if not is_web_uri(uri): logger.error("Not a URI: {}".format(uri)) sys.exit(FAILURE_EXITVAL) logger.debug("web application URI = {}".format(uri)) # First read and validate configuration file as it is mandatory argument. config = read_config(logger, args.config) if config is None: logger.error("Cannot read config file from {}".format(args.config)) sys.exit(FAILURE_EXITVAL) # Changing working directory to root will avoid problems when running # programs via sudo/su. Do this only after the config file was read # so that its path can be specified as relative. try: os.chdir("/") except OSError: logger.error("cannot change working directory to /", exc_info=True) sys.exit(FAILURE_EXITVAL) try: commands = config["commands"] except KeyError: logger.error("The config file has to contain key \"commands\"") sys.exit(FAILURE_EXITVAL) directory = args.directory if not args.directory and not args.projects and not args.indexed: # Assume directory, get the source root value from the webapp. directory = get_config_value(logger, 'sourceRoot', uri) if not directory: logger.error("Neither -d or -P or -I specified and cannot get " "source root from the webapp") sys.exit(FAILURE_EXITVAL) else: logger.info("Assuming directory: {}".format(directory)) ignore_errors = [] if args.ignore_errors: ignore_errors = args.ignore_errors else: try: ignore_errors = config["ignore_errors"] except KeyError: pass logger.debug("Ignored projects: {}".format(ignore_errors)) lock = FileLock(os.path.join(tempfile.gettempdir(), "opengrok-sync.lock")) try: with lock.acquire(timeout=0): if args.projects: dirs_to_process = args.projects logger.debug( "Processing directories: {}".format(dirs_to_process)) elif args.indexed: indexed_projects = list_indexed_projects(logger, uri) logger.debug( "Processing indexed projects: {}".format(indexed_projects)) if indexed_projects: for line in indexed_projects: dirs_to_process.append(line.strip()) else: logger.error("cannot get list of projects") sys.exit(FAILURE_EXITVAL) else: logger.debug("Processing directory {}".format(directory)) for entry in os.listdir(directory): if path.isdir(path.join(directory, entry)): dirs_to_process.append(entry) logger.debug("to process: {}".format(dirs_to_process)) cmds_base = [] for d in dirs_to_process: cmd_base = CommandSequenceBase(d, commands, args.loglevel, config.get("cleanup"), args.driveon) cmds_base.append(cmd_base) # Map the commands into pool of workers so they can be processed. with Pool(processes=int(args.workers)) as pool: try: cmds_base_results = pool.map(worker, cmds_base, 1) except KeyboardInterrupt: sys.exit(FAILURE_EXITVAL) else: for cmds_base in cmds_base_results: logger.debug( "Checking results of project {}".format(cmds_base)) cmds = CommandSequence(cmds_base) cmds.fill(cmds_base.retcodes, cmds_base.outputs, cmds_base.failed) cmds.check(ignore_errors) except Timeout: logger.warning("Already running, exiting.") sys.exit(FAILURE_EXITVAL)
class WeblateLock: """Wrapper around Redis or file based lock.""" def __init__( self, lock_path: str, scope: str, key: int, slug: str, cache_template: str = "lock:{scope}:{key}", file_template: Optional[str] = "{slug}-{scope}.lock", timeout: int = 1, ): self._timeout = timeout self._lock_path = lock_path self._scope = scope self._key = key self._slug = slug self._depth = 0 default_cache = caches["default"] self.use_redis = isinstance(default_cache, RedisCache) if self.use_redis: # Prefer Redis locking as it works distributed self._lock = Lock( default_cache.client.get_client(), name=self._format_template(cache_template), expire=5, auto_renewal=True, ) else: # Fall back to file based locking self._lock = FileLock( os.path.join(lock_path, self._format_template(file_template)), timeout=self._timeout, ) def _format_template(self, template: str): return template.format( scope=self._scope, key=self._key, slug=self._slug, ) def __enter__(self): self._depth += 1 if self._depth > 1: return if self.use_redis: try: if not self._lock.acquire(timeout=self._timeout): raise WeblateLockTimeout() except AlreadyAcquired: pass else: # Fall back to file based locking try: self._lock.acquire() except Timeout: raise WeblateLockTimeout() def __exit__(self, exc_type, exc_value, traceback): self._depth -= 1 if self._depth > 0: return try: self._lock.release() except NotAcquired: # This can happen in case of overloaded server fails to renew the # lock before expiry pass @property def is_locked(self): return bool(self._depth)
def main(): dirs_to_process = [] parser = argparse.ArgumentParser(description='Manage parallel workers.', parents=[ get_baseparser( tool_version=__version__) ]) parser.add_argument('-w', '--workers', default=multiprocessing.cpu_count(), help='Number of worker processes') # There can be only one way how to supply list of projects to process. group1 = parser.add_mutually_exclusive_group() group1.add_argument('-d', '--directory', help='Directory to process') group1.add_argument('-P', '--projects', nargs='*', help='List of projects to process') parser.add_argument('-I', '--indexed', action='store_true', help='Sync indexed projects only') parser.add_argument('-i', '--ignore_errors', nargs='*', help='ignore errors from these projects') parser.add_argument('-c', '--config', required=True, help='config file in JSON format') parser.add_argument('-U', '--uri', default='http://localhost:8080/source', help='URI of the webapp with context path') parser.add_argument('-f', '--driveon', action='store_true', default=False, help='continue command sequence processing even ' 'if one of the commands requests break') try: args = parser.parse_args() except ValueError as e: fatal(e) logger = get_console_logger(get_class_basename(), args.loglevel) uri = args.uri if not is_web_uri(uri): logger.error("Not a URI: {}".format(uri)) sys.exit(1) logger.debug("web application URI = {}".format(uri)) # First read and validate configuration file as it is mandatory argument. config = read_config(logger, args.config) if config is None: logger.error("Cannot read config file from {}".format(args.config)) sys.exit(1) # Changing working directory to root will avoid problems when running # programs via sudo/su. Do this only after the config file was read # so that its path can be specified as relative. try: os.chdir("/") except OSError: logger.error("cannot change working directory to /", exc_info=True) sys.exit(1) try: commands = config["commands"] except KeyError: logger.error("The config file has to contain key \"commands\"") sys.exit(1) directory = args.directory if not args.directory and not args.projects and not args.indexed: # Assume directory, get the source root value from the webapp. directory = get_config_value(logger, 'sourceRoot', uri) if not directory: logger.error("Neither -d or -P or -I specified and cannot get " "source root from the webapp") sys.exit(1) else: logger.info("Assuming directory: {}".format(directory)) ignore_errors = [] if args.ignore_errors: ignore_errors = args.ignore_errors else: try: ignore_errors = config["ignore_errors"] except KeyError: pass logger.debug("Ignored projects: {}".format(ignore_errors)) lock = FileLock(os.path.join(tempfile.gettempdir(), "opengrok-sync.lock")) try: with lock.acquire(timeout=0): if args.projects: dirs_to_process = args.projects logger.debug("Processing directories: {}". format(dirs_to_process)) elif args.indexed: indexed_projects = list_indexed_projects(logger, uri) logger.debug("Processing indexed projects: {}". format(indexed_projects)) if indexed_projects: for line in indexed_projects: dirs_to_process.append(line.strip()) else: logger.error("cannot get list of projects") sys.exit(1) else: logger.debug("Processing directory {}".format(directory)) for entry in os.listdir(directory): if path.isdir(path.join(directory, entry)): dirs_to_process.append(entry) logger.debug("to process: {}".format(dirs_to_process)) cmds_base = [] for d in dirs_to_process: cmd_base = CommandSequenceBase(d, commands, args.loglevel, config.get("cleanup"), args.driveon) cmds_base.append(cmd_base) # Map the commands into pool of workers so they can be processed. with Pool(processes=int(args.workers)) as pool: try: cmds_base_results = pool.map(worker, cmds_base, 1) except KeyboardInterrupt: sys.exit(1) else: for cmds_base in cmds_base_results: logger.debug("Checking results of project {}". format(cmds_base)) cmds = CommandSequence(cmds_base) cmds.fill(cmds_base.retcodes, cmds_base.outputs, cmds_base.failed) cmds.check(ignore_errors) except Timeout: logger.warning("Already running, exiting.") sys.exit(1)
conn.close() prev_price = -1 while(True): try: lock = FileLock('coinut.lock') response = urllib.urlopen(url); data = json.loads(response.read()) timestamp = long(data['timestamp']) price = float(data['tick']) if price != prev_price: if lock.acquire(): conn = sqlite3.connect('coinut.db') c = conn.cursor() with conn: values = (timestamp, price,) c.execute("INSERT INTO ticker VALUES (?,?)", values) print values conn.commit() lock.release() prev_price = price except KeyboardInterrupt:
class Store(object): auto_sync = True auto_mem_resync_counter = 0 @staticmethod def is_valid_schema(self): return all([len(x) == 3 and x[1] in DUMPERS for x in self]) def __init__(self, db_name, schema, log_level=1): """ schema example = ( (name, type, len), ... ) int str """ self._log_lvl = log_level db_name = os.path.join('db', db_name) self.lock = FileLock(db_name) if not self.is_valid_schema(schema): self.error("Invalid DB schema") raise Exception('Schema is bead!') all = self.init_mem(db_name, schema) self._memory = all self._db_name = db_name self._schema = schema def info(self, msg): if self._log_lvl > 3: print(msg) def error(self, msg): if self._log_lvl > 0: print(msg) def _insert(self, object_dict_): assert type(object_dict_) == dict, 'Wow! Not dict!' self._memory.append(object_dict_) def sync(self): self.lock.acquire() f = open(self._db_name, 'wb') for d in self._memory: for name_, type_, len_ in self._schema: bytes_bytes = DUMPERS[type_](d[name_], len_) f.write(bytes_bytes) f.write('\n') f.close() self.lock.release() self._memory = self.init_mem(self._db_name, self._schema) def execute(self, row): """ Execute SQL query. Support: select, delete, insert. """ # row = "select where value[1] == 'd' and id >= 0 limit 2" # row = "insert into ... values (2, 'awdwa')" # row = "delete where ... limit k" try: method, tail = row.split(' ', 1) method = method.lower() tail = tail.strip(' ') rez = None self.info(u'-- SQL {0} {1} --'.format(method, tail)) if method == 'insert': r = re.compile(r'^.*?values?[ ]*(\(.*?\))$', re.M) z = r.match(tail) if z: rez = [self.insert(*z.groups())] if self.auto_sync: self.sync() elif method in ['select', 'delete']: r = re.compile(r'^.*?(?:(?:where)[ ]*(.*?))?[ ]*(?:limit[ ]*(\d+))?[ ]*([dD][Ee][ScCs][ckCK])?[ ]*$') z = r.match(tail) if z: rez = self.__getattribute__('go_go')(method, z.groups()) else: rez = self.__getattribute__(method)() elif method == 'last': rez = [self.last()] if hasattr(rez, '__len__') and rez.__len__() == 1: return rez[0] return rez except Exception as e: self.error("Invalid SQL syntax detected: {0!r} by {1}".format(row, e)) raise Exception('Invalid SQL syntax!!') def go_go(self, method, args): return self.__getattribute__(method)(*args) def delete(self, where=None, limit=None, desk=None): limit = int(limit.strip()) if limit else 0 where = 'True' if not where else where where = self.fix_where(where) rez = 0 del_indexes = [] l = locals() i = 0 mem = self._memory if not desk else reversed(self._memory) for d in mem: for name_, type_, len_ in self._schema: l[name_] = d[name_] st = parser.expr(where) is_ok = eval(st.compile()) if is_ok: rez += 1 del_indexes.append(i) i += 1 if limit and rez >= limit: break z = 0 for x in sorted(del_indexes): self._delete_dy_index(x - z) z += 1 return rez def _delete_dy_index(self, index): if 0 <= index < len(self._memory): del self._memory[index] return 1 return 0 def _delete_dy_indexes(self, *indexes): del_counter = 0 for index in sorted(indexes): deleted = self._delete_dy_index(index - del_counter) del_counter += deleted return del_counter def _memory_dump(self): print '\n-- dump --' for d in self._memory: print d.values() print '-- |--| --\n' def select(self, where=None, limit=None, desk=None): limit = int(limit.strip()) if limit else 0 where = 'True' if not where else where where = self.fix_where(where) rez = [] l = locals() mem = self._memory if not desk else reversed(self._memory) for d in mem: for name_, type_, len_ in self._schema: l[name_] = d[name_] st = parser.expr(where) is_ok = eval(st.compile()) if limit and len(rez) >= limit: return rez if is_ok: rez.append(d) return rez def insert(self, insert_obj_row): if not insert_obj_row.startswith('(') or not insert_obj_row.endswith(')'): return insert_obj_row = insert_obj_row.replace("'", "'''") insert_obj_row = insert_obj_row.replace("\"", "'''") try: st = parser.expr(insert_obj_row) obj = eval(st.compile()) if type(obj) != tuple or len(obj) != len(self._schema): return d = {} i = 0 for name_, type_, len_ in self._schema: d[name_] = obj[i] i += 1 _ck = CHECKERS[type_] if _ck(d[name_]): return self._insert(d) return d except Exception as e: self.error('Insertion error!', e) return def fix_where(self, where): z = where.replace(' = ', ' == ') z = z.replace('__', '') return z.replace('import', '') def last(self): pass def init_mem(self, db_name, schema): i = 0 all = [] data = {} self.lock.acquire() def read_tail(f): while True: c = f.read(1) if not c or c == '\n': return try: _r = open(db_name, 'rb') except: self.info("Create New DB") try: t = open(db_name, 'a') t.close() except Exception as e: self.error("Can not create NEW DB", e) raise Exception('Can not create new DB') try: _r = open(db_name, 'rb') except: self.error("I/0 Error #1! Can not open!") raise Exception("I/0 Error #1! Can not open!") while 1: if i == len(schema): all.append(data) read_tail(_r) i = 0 data = {} name_, type_, len_ = schema[i] d = _r.read(len_).replace('\0', '') if not d: break zero = _r.read(1) if not zero or ord(zero) != 0: read_tail(_r) i = 0 data = {} continue try: data[name_] = type_(d) except: read_tail(_r) i = 0 data = {} else: i += 1 _r.close() self.lock.release() return all
class CandleConnector(): def __init__(self): self.lock = FileLock("config.csv.lock") # make dict here that stores the amount for each coin self.config = "config.csv" self.candles = candles.BinaceConnector() def readConfig(self): self.lock.acquire() df = pd.read_csv(self.config, encoding='utf8', delimiter=',', names=[ 'coin', 'capital', 'starting', 'limit', 'currentPrice', 'autobought', 'takeprofit', 'updatetime', 'orderid' ]) self.lock.release() df.set_index('coin', inplace=True) return df #get the current config def getCoinConfigData(self, coin): df = self.readConfig() return df.loc[coin] def getBuyPower(self): return float(self.candles.getUSD()) #save a new copy of the config def setCoinConfigData(self, df): self.lock.acquire() df.to_csv(f'config.csv', mode='w', header=False, index=True) self.lock.release() def getAutoBoughtAmount(self, coin): return float(self.getCoinConfigData(coin)['autobought']) # helper for buying a number of coins at current price def orderNumber(self, coin, number): return (self.candles.buyMarket(coin, number)) # gives you a quote for a coin def getQuote(self, coin): return float(self.candles.getCoinPrice(coin)) # set an order for an number amount def orderAmount(self, coin, amount): return (self.candles.order_buy_crypto_by_price(coin, amount)) # write out to a log file def logit(self, message, destination): with open(f"logdata/{destination}.txt", "a") as f: f.write(message) f.write("\n") def saveCoinBuyData(self, coin, price, amount, setcap=None): df = self.readConfig() if setcap is not None: df.at[coin, 'capital'] = setcap df.at[coin, 'starting'] = price df.at[coin, 'autobought'] = amount df.at[coin, 'limit'] = price * .98 self.setCoinConfigData(df) # check to see how much can be purchased with the current capital # then purchase that amount of coins def buyNow(self, coin, strat=None): coinsCapital = self.getCoinConfigData(coin)['capital'] avalFunds = self.getBuyPower() if (coinsCapital > avalFunds) is True: return 0 price = self.getQuote(coin) #TODO add logic that allows for multiple strategies that will #allow for different allocations of the starting capital BOUGHT = float(coinsCapital / self.getQuote(coin)) minOrder = None minNot = None print(BOUGHT) #grab the trading rules for the coin for filt in (self.candles.getCoinInfo(coin)['filters']): if filt['filterType'] == "LOT_SIZE": minOrder = float(filt['minQty']) if filt['filterType'] == 'MIN_NOTIONAL': minNot = float(filt['minNotional']) mod = BOUGHT % minOrder #make sure the amount we are buying is standardized for Binance if mod: BOUGHT = BOUGHT - mod #this needs to get the perciesion from the filter BOUGHT = round(BOUGHT, 8) print(BOUGHT) if (BOUGHT * price) > minNot: order = self.orderNumber(coin, BOUGHT) self.saveCoinBuyData(coin, price, BOUGHT) self.logit(f"BUYING {order}", coin) else: BOUGHT = None self.logit( f"Failed to buy {BOUGHT}, {coin}. Due minNotional of {minNot}", coin) return BOUGHT #sell an amount at current price def sellNow(self, coin): #get the amount the bot bought amount = self.getAutoBoughtAmount(coin) if amount > 0: # self.candles.testOrder(coin, SIDE_SELL, amount) sellorder = self.candles.sellMarket(coin, amount) orderID = sellorder['clientOrderId'] status = self.candles.checkStatus(coin, orderID) timeout = 5 time.sleep(2) #check a couple of times to make sure we are selling while status != 'FILLED': if timeout > 5: timeout = 0 self.candles.cancelOrder(coin, orderID) status = self.candles.checkStatus(coin, orderID) timeout += 1 time.sleep(2) # save the data for analysis later and reset the bot coin's config self.logit(f"SELLING DUE TO STRAT {sellorder}", coin) sellprice = float(sellorder['fills'][0]['price']) * amount print(sellprice) self.saveCoinBuyData(coin, 0, 0, setcap=sellprice)
def main(): ret = 0 parser = argparse.ArgumentParser(description='project mirroring', parents=[get_baseparser( tool_version=__version__) ]) parser.add_argument('project') parser.add_argument('-c', '--config', help='config file in JSON/YAML format') parser.add_argument('-U', '--uri', default='http://localhost:8080/source', help='uri of the webapp with context path') parser.add_argument('-b', '--batch', action='store_true', help='batch mode - will log into a file') parser.add_argument('-B', '--backupcount', default=8, help='how many log files to keep around in batch mode') parser.add_argument('-I', '--incoming', action='store_true', help='Check for incoming changes, terminate the ' 'processing if not found.') try: args = parser.parse_args() except ValueError as e: print_exc_exit(e) logger = get_console_logger(get_class_basename(), args.loglevel) if args.config: config = read_config(logger, args.config) if config is None: logger.error("Cannot read config file from {}".format(args.config)) sys.exit(1) else: config = {} GLOBAL_TUNABLES = [HOOKDIR_PROPERTY, PROXY_PROPERTY, LOGDIR_PROPERTY, COMMANDS_PROPERTY, PROJECTS_PROPERTY, HOOK_TIMEOUT_PROPERTY, CMD_TIMEOUT_PROPERTY] diff = diff_list(config.keys(), GLOBAL_TUNABLES) if diff: logger.error("unknown global configuration option(s): '{}'" .format(diff)) sys.exit(1) # Make sure the log directory exists. logdir = config.get(LOGDIR_PROPERTY) if logdir: check_create_dir(logger, logdir) uri = args.uri if not is_web_uri(uri): logger.error("Not a URI: {}".format(uri)) sys.exit(1) logger.debug("web application URI = {}".format(uri)) source_root = get_config_value(logger, 'sourceRoot', uri) if not source_root: sys.exit(1) logger.debug("Source root = {}".format(source_root)) project_config = None projects = config.get(PROJECTS_PROPERTY) if projects: if projects.get(args.project): project_config = projects.get(args.project) else: for proj in projects.keys(): try: pattern = re.compile(proj) except re.error: logger.error("Not a valid regular expression: {}". format(proj)) continue if pattern.match(args.project): logger.debug("Project '{}' matched pattern '{}'". format(args.project, proj)) project_config = projects.get(proj) break hookdir = config.get(HOOKDIR_PROPERTY) if hookdir: logger.debug("Hook directory = {}".format(hookdir)) command_timeout = get_int(logger, "command timeout", config.get(CMD_TIMEOUT_PROPERTY)) if command_timeout: logger.debug("Global command timeout = {}".format(command_timeout)) hook_timeout = get_int(logger, "hook timeout", config.get(HOOK_TIMEOUT_PROPERTY)) if hook_timeout: logger.debug("Global hook timeout = {}".format(hook_timeout)) prehook = None posthook = None use_proxy = False ignored_repos = None if project_config: logger.debug("Project '{}' has specific (non-default) config". format(args.project)) # Quick sanity check. KNOWN_PROJECT_TUNABLES = [DISABLED_PROPERTY, CMD_TIMEOUT_PROPERTY, HOOK_TIMEOUT_PROPERTY, PROXY_PROPERTY, IGNORED_REPOS_PROPERTY, HOOKS_PROPERTY] diff = diff_list(project_config.keys(), KNOWN_PROJECT_TUNABLES) if diff: logger.error("unknown project configuration option(s) '{}' " "for project {}".format(diff, args.project)) sys.exit(1) project_command_timeout = get_int(logger, "command timeout for " "project {}". format(args.project), project_config. get(CMD_TIMEOUT_PROPERTY)) if project_command_timeout: command_timeout = project_command_timeout logger.debug("Project command timeout = {}". format(command_timeout)) project_hook_timeout = get_int(logger, "hook timeout for " "project {}". format(args.project), project_config. get(HOOK_TIMEOUT_PROPERTY)) if project_hook_timeout: hook_timeout = project_hook_timeout logger.debug("Project hook timeout = {}". format(hook_timeout)) ignored_repos = project_config.get(IGNORED_REPOS_PROPERTY) if ignored_repos: if not isinstance(ignored_repos, list): logger.error("{} for project {} is not a list". format(IGNORED_REPOS_PROPERTY, args.project)) sys.exit(1) logger.debug("has ignored repositories: {}". format(ignored_repos)) hooks = project_config.get(HOOKS_PROPERTY) if hooks: if not hookdir: logger.error("Need to have '{}' in the configuration " "to run hooks".format(HOOKDIR_PROPERTY)) sys.exit(1) if not os.path.isdir(hookdir): logger.error("Not a directory: {}".format(hookdir)) sys.exit(1) for hookname in hooks: if hookname == "pre": prehook = hookpath = os.path.join(hookdir, hooks['pre']) logger.debug("pre-hook = {}".format(prehook)) elif hookname == "post": posthook = hookpath = os.path.join(hookdir, hooks['post']) logger.debug("post-hook = {}".format(posthook)) else: logger.error("Unknown hook name {} for project {}". format(hookname, args.project)) sys.exit(1) if not is_exe(hookpath): logger.error("hook file {} does not exist or not " "executable".format(hookpath)) sys.exit(1) if project_config.get(PROXY_PROPERTY): if not config.get(PROXY_PROPERTY): logger.error("global proxy setting is needed in order to" "have per-project proxy") sys.exit(1) logger.debug("will use proxy") use_proxy = True if not ignored_repos: ignored_repos = [] # Log messages to dedicated log file if running in batch mode. if args.batch: if not logdir: logger.error("The logdir property is required in batch mode") sys.exit(1) logfile = os.path.join(logdir, args.project + ".log") logger.debug("Switching logging to the {} file". format(logfile)) logger = logger.getChild("rotating") logger.setLevel(args.loglevel) logger.propagate = False handler = RotatingFileHandler(logfile, maxBytes=0, mode='a', backupCount=args.backupcount) formatter = logging.Formatter("%(asctime)s - %(levelname)s: " "%(message)s", '%m/%d/%Y %I:%M:%S %p') handler.setFormatter(formatter) handler.doRollover() logger.addHandler(handler) # We want this to be logged to the log file (if any). if project_config: if project_config.get(DISABLED_PROPERTY): logger.info("Project {} disabled, exiting". format(args.project)) sys.exit(CONTINUE_EXITVAL) lock = FileLock(os.path.join(tempfile.gettempdir(), args.project + "-mirror.lock")) try: with lock.acquire(timeout=0): proxy = config.get(PROXY_PROPERTY) if use_proxy else None # # Cache the repositories first. This way it will be known that # something is not right, avoiding any needless pre-hook run. # repos = [] try: repos = get_repos_for_project(logger, args.project, ignored_repos, commands=config. get(COMMANDS_PROPERTY), proxy=proxy, command_timeout=command_timeout, source_root=source_root, uri=uri) except RepositoryException as ex: logger.error('failed to get repositories for project {}: {}'. format(args.project, ex)) sys.exit(1) if not repos: logger.info("No repositories for project {}". format(args.project)) sys.exit(CONTINUE_EXITVAL) # Check if any of the repositories contains incoming changes. if args.incoming: got_incoming = False for repo in repos: try: if repo.incoming(): logger.debug('Repository {} has incoming changes'. format(repo)) got_incoming = True break except RepositoryException: logger.error('Cannot determine incoming changes for ' 'repository {}'.format(repo)) sys.exit(1) if not got_incoming: logger.info('No incoming changes for repositories in ' 'project {}'. format(args.project)) sys.exit(CONTINUE_EXITVAL) if prehook: logger.info("Running pre hook") if run_hook(logger, prehook, os.path.join(source_root, args.project), proxy, hook_timeout) != 0: logger.error("pre hook failed for project {}". format(args.project)) logging.shutdown() sys.exit(1) # # If one of the repositories fails to sync, the whole project sync # is treated as failed, i.e. the program will return 1. # for repo in repos: logger.info("Synchronizing repository {}". format(repo.path)) if repo.sync() != 0: logger.error("failed to sync repository {}". format(repo.path)) ret = 1 if posthook: logger.info("Running post hook") if run_hook(logger, posthook, os.path.join(source_root, args.project), proxy, hook_timeout) != 0: logger.error("post hook failed for project {}". format(args.project)) logging.shutdown() sys.exit(1) except Timeout: logger.warning("Already running, exiting.") sys.exit(1) logging.shutdown() sys.exit(ret)
f = f[:ind] + '.png' problems.append((f, reason)) if not checking: numImages = min(numImages, NUM_PER_GROUP) if autochecking: files = problems if numDone >= numImages: groupsDone[groupIndex] = True if startHere is None and not checking and not doTrans: #import pdb; pdb.set_trace() continue #print 'group '+groupName+', template image: '+imageTemplate #templateFile=os.path.join(directory,groupName,template) lock = FileLock(template, timeout=None) try: lock.acquire() textsT = fieldsT = pairsT = samePairsT = horzLinksT = groupsT = transcriptionsT = cornersT = cornersActualT = None if template is not None: with open(template) as f: read = json.load(f) textsT = read['textBBs'] fieldsT = read['fieldBBs'] pairsT = read['pairs'] samePairsT = read['samePairs'] groupsT = read['groups'] if 'transcriptions' in read: transcriptionsT = read['transcriptions'] cornersT = read['page_corners'] if 'horzLinks' in read and not skipHLinks: horzLinksT = read['horzLinks'] #cornersActualT=read['actualPage_corners']
def main(argv): current_ver = '' start_ver = '' version_list = [] rev_version_list = [] # Version list in reverse order fltg_root_dir = None try: opts, args = getopt.getopt(argv, "hv:s:d:", ["help"]) except getopt.GetoptError: usage() sys.exit(2) for opt, arg in opts: if opt in ("-h", "--help"): usage() sys.exit(0); elif opt == '-v': current_ver = arg elif opt == '-s': start_ver = arg elif opt == '-d': fltg_root_dir = arg else: usage() sys.exit(0); if start_ver == '': start_ver = current_ver try: sdk = os.environ['SDK'] except: print("SDK Environment variable must be defined") sys.exit(0); try: issu_db_dir = os.environ['ISSU_DB'] + '/' except: issu_db_dir = sdk + ISSU_DB_DEF_DIR base_dir = issu_db_dir ver_list_c = issu_version_list.issu_version_list(base_dir, start_ver, current_ver) rev_version_list = ver_list_c.get_ver_list() version_list = ver_list_c.get_ver_list(reverse = False) # Build the base dir if current_ver == '': base_dir += 'current' else: base_dir += 'ver_' + current_ver target_dir = base_dir # Lock the DB to enable multiple simultaneous builds. If the lock is taken # it means that other build is building the DB. Let it finish first before # we can jump to the next phase of compiling the DLL lock_file = issu_db_dir + 'issu_db.lock' try: lock = FileLock(lock_file, 1800) lock.acquire() # wait up to 30 min except Timeout: print('Failed to acquire ISSU DB lock') sys.exit(-1) db_class = issu_db_gen.global_struct_db(base_dir, None) if not db_class.generate_c_db(current_ver, version_list): print('Failed to generate DB for version %s' % current_ver) base_dir = issu_db_dir + 'ver_' for ver in rev_version_list: if ver != current_ver: db_class.parse_previous_versions(base_dir, ver) # Final generation and shutdown the DB gen class db_class.shutdown() # Save reg file signatures into the current version DB if current_ver != '': if fltg_root_dir == None: fltg_root_dir = sdk + FLTG_INPUT_DIR reg_file_c = issu_reg_files.reg_file_sig(issu_db_dir) reg_file_c.save_sig(current_ver, fltg_root_dir) # Generate the patch list patch_class = issu_patch_insert.issu_patch(issu_db_dir) patch_class.read_patch_db(version_list) patch_class.create_patch_c_file() patch_class.create_patch_makefile() lock.release()
class Client(JSONFile, MioLogger): base_dir: Runtime[Union[Path, str]] server: Union[URL, str] = "" device_id: str = "" user_id: UserId = "" # type: ignore access_token: str = "" net: Runtime[Network] = field(init=False, repr=False) auth: Runtime[Auth] = field(init=False, repr=False) profile: Runtime[Profile] = field(init=False, repr=False) rooms: Runtime[Rooms] = field(init=False, repr=False) sync: Runtime[Sync] = field(init=False, repr=False) e2e: Runtime[E2E] = field(init=False, repr=False) devices: Runtime[Devices] = field(init=False, repr=False) media: Runtime[MediaStore] = field(init=False, repr=False) _filters: Runtime[FilterStore] = field(init=False, repr=False) _lock: Runtime[Optional[FileLock]] = field(init=False, repr=False) _terminated: Runtime[bool] = field(init=False, repr=False) def __post_init__(self) -> None: MioLogger.__post_init__(self) self.net = Network(self) self.auth = Auth(self) self.profile = Profile(self) self.account_data = AccountData(self) self.rooms = Rooms(self) self.sync = Sync(self) self.e2e = E2E(self) self.devices = Devices(self) self.media = MediaStore(self) self._filters = FilterStore(self) self._lock = None self._terminated = False JSONFile.__post_init__(self) async def __aenter__(self) -> "Client": return self async def __aexit__(self, exc_type, exc, tb) -> None: await self.terminate() def __del__(self) -> None: if self._lock: self._lock.release() @property def path(self) -> AsyncPath: return AsyncPath(self.base_dir) / "client.json" @property async def was_saved(self) -> bool: data = "{}" with suppress(FileNotFoundError): data = await self.path.read_text() return bool(json.loads(data).get("access_token")) async def load(self, **base_dir_placeholders: str) -> "Client": self.base_dir = str(self.base_dir).format(**base_dir_placeholders) await super().load() self._acquire_lock() for attr in self.__dict__.values(): if isinstance(attr, ClientModule): await attr.load() return self async def terminate(self) -> None: self.__del__() # release lock await self.net.disconnect() self._terminated = True def _acquire_lock(self) -> None: self._lock = FileLock(str(Path(self.base_dir) / ".lock")) self._lock.acquire(timeout=1)
class MultiprocessRotatingFileHandler(TimedRotatingFileHandler): """ Ref: https://github.com/di/mrfh/blob/master/mrfh/__init__.py * 修改doRollover逻辑,避免日志分片时删除已分片日志 * 释放日志锁时关闭FileStream,解决文件重命名操作被拒绝的问题 * 频繁reopen FileStream,造成严重的性能损耗 * 使用FileLock替代threading.Lock,单进程条件下存在性能损失 """ def __init__(self, lock_file, *args, **kwargs): self.file_lock = FileLock(lock_file) super(MultiprocessRotatingFileHandler, self).__init__(*args, **kwargs) def _open_file(self): self.stream = self._open() def acquire(self): self.file_lock.acquire() if self.stream and self.stream.closed: self._open_file() def release(self): if self.stream and not self.stream.closed: self.stream.flush() self.stream.close() self.file_lock.release() def close(self): if self.stream and not self.stream.closed: self.stream.flush() self.stream.close() if self.file_lock.is_locked: self.file_lock.release() def doRollover(self): if self.stream: self.stream.close() self.stream = None # get the time that this sequence started at and make it a TimeTuple current_time = int(time.time()) dst_now = time.localtime(current_time)[-1] t = self.rolloverAt - self.interval if self.utc: time_tuple = time.gmtime(t) else: time_tuple = time.localtime(t) dst_then = time_tuple[-1] if dst_now != dst_then: if dst_now: addend = 3600 else: addend = -3600 time_tuple = time.localtime(t + addend) dfn = self.rotation_filename(self.baseFilename + "." + time.strftime(self.suffix, time_tuple)) # # Changed part # if os.path.exists(dfn): # os.remove(dfn) if not os.path.exists(dfn) and os.path.exists(self.baseFilename): self.rotate(self.baseFilename, dfn) # # Changed part end if self.backupCount > 0: for s in self.getFilesToDelete(): os.remove(s) if not self.delay: self.stream = self._open() new_rollover_at = self.computeRollover(current_time) while new_rollover_at <= current_time: new_rollover_at = new_rollover_at + self.interval # If DST changes and midnight or weekly rollover, adjust for this. if (self.when == 'MIDNIGHT' or self.when.startswith('W')) and not self.utc: dst_at_rollover = time.localtime(new_rollover_at)[-1] if dst_now != dst_at_rollover: if not dst_now: # DST kicks in before next rollover, so we need to deduct an hour addend = -3600 else: # DST bows out before next rollover, so we need to add an hour addend = 3600 new_rollover_at += addend self.rolloverAt = new_rollover_at
def main(): parser = argparse.ArgumentParser(description='project management.', formatter_class=argparse. ArgumentDefaultsHelpFormatter, parents=[get_baseparser( tool_version=__version__) ]) parser.add_argument('-b', '--base', default="/var/opengrok", help='OpenGrok instance base directory') parser.add_argument('-R', '--roconfig', help='OpenGrok read-only configuration file') parser.add_argument('-U', '--uri', default='http://localhost:8080/source', help='URI of the webapp with context path') parser.add_argument('-c', '--configmerge', help='path to the ConfigMerge binary') parser.add_argument('--java', help='Path to java binary ' '(needed for config merge program)') parser.add_argument('-j', '--jar', help='Path to jar archive to run') parser.add_argument('-u', '--upload', action='store_true', help='Upload configuration at the end') parser.add_argument('-n', '--noop', action='store_true', default=False, help='Do not run any commands or modify any config' ', just report. Usually implies ' 'the --debug option.') parser.add_argument('-N', '--nosourcedelete', action='store_true', default=False, help='Do not delete source code when ' 'deleting a project') group = parser.add_mutually_exclusive_group() group.add_argument('-a', '--add', metavar='project', nargs='+', help='Add project (assumes its source is available ' 'under source root') group.add_argument('-d', '--delete', metavar='project', nargs='+', help='Delete project and its data and source code') group.add_argument('-r', '--refresh', action='store_true', help='Refresh configuration. If read-only ' 'configuration is supplied, it is merged ' 'with current ' 'configuration.') try: args = parser.parse_args() except ValueError as e: fatal(e) doit = not args.noop configmerge = None # # Setup logger as a first thing after parsing arguments so that it can be # used through the rest of the program. # logger = get_console_logger(get_class_basename(), args.loglevel) if args.nosourcedelete and not args.delete: logger.error("The no source delete option is only valid for delete") sys.exit(1) # Set the base directory if args.base: if path.isdir(args.base): logger.debug("Using {} as instance base". format(args.base)) else: logger.error("Not a directory: {}\n" "Set the base directory with the --base option." .format(args.base)) sys.exit(1) # If read-only configuration file is specified, this means read-only # configuration will need to be merged with active webapp configuration. # This requires config merge tool to be run so couple of other things # need to be checked. if args.roconfig: if path.isfile(args.roconfig): logger.debug("Using {} as read-only config".format(args.roconfig)) else: logger.error("File {} does not exist".format(args.roconfig)) sys.exit(1) configmerge_file = get_command(logger, args.configmerge, "opengrok-config-merge") if configmerge_file is None: logger.error("Use the --configmerge option to specify the path to" "the config merge script") sys.exit(1) configmerge = [configmerge_file] if args.loglevel: configmerge.append('-l') configmerge.append(str(args.loglevel)) if args.jar is None: logger.error('jar file needed for config merge tool, ' 'use --jar to specify one') sys.exit(1) uri = args.uri if not is_web_uri(uri): logger.error("Not a URI: {}".format(uri)) sys.exit(1) logger.debug("web application URI = {}".format(uri)) lock = FileLock(os.path.join(tempfile.gettempdir(), os.path.basename(sys.argv[0]) + ".lock")) try: with lock.acquire(timeout=0): if args.add: for proj in args.add: project_add(doit=doit, logger=logger, project=proj, uri=uri) config_refresh(doit=doit, logger=logger, basedir=args.base, uri=uri, configmerge=configmerge, jar_file=args.jar, roconfig=args.roconfig, java=args.java) elif args.delete: for proj in args.delete: project_delete(logger=logger, project=proj, uri=uri, doit=doit, deletesource=not args.nosourcedelete) config_refresh(doit=doit, logger=logger, basedir=args.base, uri=uri, configmerge=configmerge, jar_file=args.jar, roconfig=args.roconfig, java=args.java) elif args.refresh: config_refresh(doit=doit, logger=logger, basedir=args.base, uri=uri, configmerge=configmerge, jar_file=args.jar, roconfig=args.roconfig, java=args.java) else: parser.print_help() sys.exit(1) if args.upload: main_config = get_config_file(basedir=args.base) if path.isfile(main_config): if doit: with io.open(main_config, mode='r', encoding="utf-8") as config_file: config_data = config_file.read().encode("utf-8") if not set_configuration(logger, config_data, uri): sys.exit(1) else: logger.error("file {} does not exist".format(main_config)) sys.exit(1) except Timeout: logger.warning("Already running, exiting.") sys.exit(1)
class CustomLogGui(wx.LogGui): """Logger.""" def __init__(self, log_file, debug=False): """Initialize.""" self.format = "%(message)s" self.file_name = log_file self.file_lock = FileLock(self.file_name + '.lock') self.debug = debug try: with self.file_lock.acquire(1): with codecs.open(self.file_name, "w", "utf-8") as f: f.write("") except Exception: self.file_name = None wx.LogGui.__init__(self) def DoLogText(self, msg): """Log the text.""" try: if self.file_name is not None: with self.file_lock.acquire(1): with codecs.open(self.file_name, 'a', encoding='utf-8') as f: f.write(msg) else: msg = "[ERROR] Could not acquire lock for log!\n" + msg except Exception: self.file_name = None if self.debug: sys.stdout.write( (self.format % {"message": msg}) ) wx.LogGui.DoLogText(self, msg) def DoLogTextAtLevel(self, level, msg): """Perform log at level.""" current = self.GetLogLevel() if level <= current and level == wx.LOG_Info: self._debug(msg) elif level <= current and level == wx.LOG_FatalError: self._critical(msg) elif level <= current and level == wx.LOG_Warning: self._warning(msg) elif level <= current and level == wx.LOG_Error: self._error(msg) def formatter(self, lvl, log_fmt, msg, msg_fmt=None): """Special formatters for log message.""" return log_fmt % { "loglevel": lvl, "message": util.to_ustr(msg if msg_fmt is None else msg_fmt(msg)) } def _log(self, msg): """Base logger.""" return self.format % {"message": msg} def _debug(self, msg, log_fmt="%(loglevel)s: %(message)s\n"): """Debug level log.""" self.DoLogText(self._log(self.formatter("DEBUG", log_fmt, msg))) def _critical(self, msg, log_fmt="%(loglevel)s: %(message)s\n"): """Critical level log.""" self.DoLogText(self._log(self.formatter("CRITICAL", log_fmt, msg))) def _warning(self, msg, log_fmt="%(loglevel)s: %(message)s\n"): """Warning level log.""" self.DoLogText(self._log(self.formatter("WARNING", log_fmt, msg))) def _error(self, msg, log_fmt="%(loglevel)s: %(message)s\n"): """Error level log.""" self.DoLogText(self._log(self.formatter("ERROR", log_fmt, msg)))