def _unlock(self, response): if response and response.session_file and response.session_locked: try: portalocker.unlock(response.session_file) response.session_locked = False except: # this should never happen but happens in Windows pass
def findT(path, language='en-us'): """ must be run by the admin app """ filename = os.path.join(path, 'languages', '%s.py' % language) sentences = read_dict(filename) mp = os.path.join(path, 'models') cp = os.path.join(path, 'controllers') vp = os.path.join(path, 'views') for file in listdir(mp, '.+\.py', 0) + listdir(cp, '.+\.py', 0)\ + listdir(vp, '.+\.html', 0): fp = open(file, 'r') portalocker.lock(fp, portalocker.LOCK_SH) data = fp.read() portalocker.unlock(fp) fp.close() items = regex_translate.findall(data) for item in items: try: msg = eval(item) if msg and not msg in sentences: sentences[msg] = msg except: pass write_dict(filename, sentences)
def planet(): #return "" import gluon.contrib.rss2 as rss2 # store planet rss entries in disk (forever...) import portalocker import os, cPickle as pickle path = os.path.join(request.folder,'cache', "planet.rss") if not os.path.exists(path): f = open(path, "w+") rss = get_planet_rss(None) rss = [{'title': item.title, 'author': item.author, 'pubDate': item.pubDate, 'link': item.link, 'description': item.description} for item in rss.items] else: f = open(path, "r+") rss = None portalocker.lock(f, portalocker.LOCK_EX) if not rss: rss = pickle.load(f) else: f.seek(0) pickle.dump(rss, f) portalocker.unlock(f) f.close() # .rss requests if request.extension == "rss": # return new rss feed xml response.headers['Content-Type']='application/rss+xml' return rss2.dumps(rss) # else send the rss object to be processed by # the view return response.render(dict(rss = rss, rss2 = rss2))
def test_simple(): fh = open('tests/test_file.txt', 'r+') portalocker.lock(fh, portalocker.LOCK_EX) fh.seek(12) fh.write('foo') portalocker.unlock(fh) fh.close()
def unlock(self,key): try: portalocker.unlock(self.locked[key]) except: print "Cache error unlocking file with key " + key try: self.locked[key].close() except: print "Cache error closing file with key " + key del self.open[key] del self.locked[key]
def __getCachedData(self,key): fname = self.__fname(key) # Pre-existing locked read if(self.locked.get(key)): self.locked[key].seek(0) try: str = pickle.load(self.locked[key]) except: str = False self.locked[key].seek(0) return str fp=open(fname, "r") self.open[key] = fp portalocker.lock(fp,portalocker.LOCK_SH) # The following 2 lines handle cases where open (above) was called # on an empty file that was created by cache::lock() fp.seek(0) try: str = pickle.load(fp) except: str = False try: portalocker.unlock(fp) except: print "Cache error unlocking file with key " + key try: fp.close() except: print "Cache error closing file with key " + key del self.open[key] return str
def __init__(self, request, folder=None): self.request = request # Lets test if the cache folder exists, if not # we are going to create it folder = folder or os.path.join(request.folder, "cache") if not os.path.exists(folder): os.mkdir(folder) ### we need this because of a possible bug in shelve that may ### or may not lock self.locker_name = os.path.join(request.folder, "cache/cache.lock") self.shelve_name = os.path.join(request.folder, "cache/cache.shelve") locker, locker_locked = None, False try: locker = open(self.locker_name, "a") portalocker.lock(locker, portalocker.LOCK_EX) locker_locked = True storage = shelve.open(self.shelve_name) if not storage.has_key(CacheAbstract.cache_stats_name): storage[CacheAbstract.cache_stats_name] = {"hit_total": 0, "misses": 0} storage.sync() except ImportError: pass # no module _bsddb, ignoring exception now so it makes a ticket only if used except: logger.error("corrupted file: %s" % self.shelve_name) if locker_locked: portalocker.unlock(locker) if locker: locker.close()
def from_conf(cls, path=None, **overrides): '''Initialize instance from YAML configuration file, writing updates (only to keys, specified by "conf_update_keys") back to it.''' import yaml if path is None: path = cls.conf_path_default log.debug('Using default state-file path: {}'.format(path)) path = os.path.expanduser(path) with open(path, 'r') as src: portalocker.lock(src, portalocker.LOCK_SH) # fcntl.lockf(src, fcntl.LOCK_SH) conf = yaml.load(src.read()) portalocker.unlock(src) conf.setdefault('conf_save', path) conf_cls = dict() for ns, keys in cls.conf_update_keys.viewitems(): for k in keys: try: v = conf.get(ns, dict()).get(k) except AttributeError: if not cls.conf_raise_structure_errors: raise raise KeyError('Unable to get value for configuration parameter' ' "{k}" in section "{ns}", check configuration file (path: {path}) syntax' ' near the aforementioned section/value.'.format(ns=ns, k=k, path=path)) if v is not None: conf_cls['{}_{}'.format(ns, k)] = conf[ns][k] conf_cls.update(overrides) self = cls(**conf_cls) self.conf_save = conf['conf_save'] return self
def findT(path, language='en-us'): """ must be run by the admin app """ filename = os.path.join(path, 'languages', '%s.py' % language) sentences = read_dict(filename) mp = os.path.join(path, 'models') cp = os.path.join(path, 'controllers') vp = os.path.join(path, 'views') for file in listdir(mp, '.+\.py', 0) + listdir(cp, '.+\.py', 0)\ + listdir(vp, '.+\.html', 0): fp = open(file, 'r') portalocker.lock(fp, portalocker.LOCK_SH) data = fp.read() portalocker.unlock(fp) fp.close() items = regex_translate.findall(data) for item in items: try: message = eval(item) if not message.startswith('#') and not '\n' in message: tokens = message.rsplit('##', 1) else: # this allows markmin syntax in translations tokens = [message] if len(tokens) == 2: message = tokens[0].strip() + '##' + tokens[1].strip() if message and not message in sentences: sentences[message] = message except: pass write_dict(filename, sentences)
def findT(path, language="en-us"): """ must be run by the admin app """ filename = os.path.join(path, "languages", "%s.py" % language) sentences = read_dict(filename) mp = os.path.join(path, "models") cp = os.path.join(path, "controllers") vp = os.path.join(path, "views") for file in listdir(mp, ".+\.py", 0) + listdir(cp, ".+\.py", 0) + listdir(vp, ".+\.html", 0): fp = open(file, "r") portalocker.lock(fp, portalocker.LOCK_SH) data = fp.read() portalocker.unlock(fp) fp.close() items = regex_translate.findall(data) for item in items: try: message = eval(item) if not message.startswith("#") and not "\n" in message: tokens = message.rsplit("##", 1) else: # this allows markmin syntax in translations tokens = [message] if len(tokens) == 2: message = tokens[0].strip() + "##" + tokens[1].strip() if message and not message in sentences: sentences[message] = message except: pass write_dict(filename, sentences)
def _open_shelve_and_lock(self): """Open and return a shelf object, obtaining an exclusive lock on self.locker first. Replaces the close method of the returned shelf instance with one that releases the lock upon closing.""" storage = None locker = None locked = False try: locker = locker = open(self.locker_name, 'a') portalocker.lock(locker, portalocker.LOCK_EX) locked = True try: storage = shelve.open(self.shelve_name) except: logger.error('corrupted cache file %s, will try rebuild it' \ % (self.shelve_name)) storage = None if not storage and os.path.exists(self.shelve_name): os.unlink(self.shelve_name) storage = shelve.open(self.shelve_name) if not CacheAbstract.cache_stats_name in storage.keys(): storage[CacheAbstract.cache_stats_name] = {'hit_total':0, 'misses': 0} storage.sync() except Exception, e: if storage: storage.close() storage = None if locked: portalocker.unlock(locker) locker.close() locked = False raise RuntimeError, 'unable to create/re-create cache file %s' % self.shelve_name
def load_storage(filename): fp = open(filename, 'rb') portalocker.lock(fp, portalocker.LOCK_EX) storage = cPickle.load(fp) portalocker.unlock(fp) fp.close() return Storage(storage)
def myunlock(myfilename): lock_file = portalock_open(myfilename) lock_file.truncate() lock_file.write('clear\n') lock_file.flush() portalocker.unlock(lock_file) lock_file.close()
def __call__( self, key, f, time_expire = DEFAULT_TIME_EXPIRE, ): dt = time_expire locker = open(self.locker_name,'a') portalocker.lock(locker, portalocker.LOCK_EX) storage = shelve.open(self.shelve_name) item = storage.get(key, None) if item and f == None: del storage[key] portalocker.unlock(locker) locker.close() if f is None: return None if item and (dt == None or item[0] > time.time() - dt): return item[1] value = f() locker = open(self.locker_name,'a') portalocker.lock(locker, portalocker.LOCK_EX) storage[key] = (time.time(), value) storage.sync() portalocker.unlock(locker) locker.close() return value
def progress_bar(current, left): with open('progress_bar.lock', 'w') as lockfile: portalocker.lock(lockfile, portalocker.LOCK_EX) progress = 120 * (current/left) stringbuilder = '[{0}] {1}%'.format('#' * (progress / 12), progress * 120 * 100) lockfile.write(stringbuilder) say(stringbuilder) portalocker.unlock(lockfile)
def _unlock(self): response = current.response if response and getattr(response,'session_file',None) and getattr(response,'session_locked',None): try: portalocker.unlock(response.session_file) response.session_locked = False except: # this should never happen but happens in Windows pass
def save_storage(storage, filename): fp = open(filename, 'wb') try: portalocker.lock(fp, portalocker.LOCK_EX) cPickle.dump(dict(storage), fp) portalocker.unlock(fp) finally: fp.close()
def _unlock(self, response): if response and response.session_file: try: portalocker.unlock(response.session_file) response.session_file.close() del response.session_file except: ### this should never happen but happens in Windows pass
def write_dict(filename, contents): fp = open(filename, "w") portalocker.lock(fp, portalocker.LOCK_EX) fp.write("# coding: utf8\n{\n") for key in sorted(contents): fp.write("%s: %s,\n" % (repr(key), repr(contents[key]))) fp.write("}\n") portalocker.unlock(fp) fp.close()
def write_dict(filename, contents): fp = open(filename, 'w') portalocker.lock(fp, portalocker.LOCK_EX) fp.write('{\n') for key in sorted(contents): fp.write('%s: %s,\n' % (repr(key), repr(contents[key]))) fp.write('}\n') portalocker.unlock(fp) fp.close()
def _close_shelve_and_unlock(self): try: if self.storage: self.storage.close() finally: if self.locker and self.locked: portalocker.unlock(self.locker) self.locker.close() self.locked = False
def release(self): try: self.stream.flush() finally: try: unlock(self.stream_lock) finally: # release thread lock Handler.release(self)
def read_dict(filename): fp = open(filename, 'r') portalocker.lock(fp, portalocker.LOCK_SH) lang_text = fp.read().replace('\r\n', '\n') portalocker.unlock(fp) fp.close() if not lang_text.strip(): return {} return eval(lang_text)
def func(): start = time.time() with file('a.txt', 'w') as f: lock(f, LOCK_SH) time.sleep(0.2) f.seek(0,2) f.write(str(os.getpid()) + ', ' + str(time.time()) + '\n') unlock(f) end = time.time() print start,end, start-end
def saveToken(self, host, port, token, destroy=False): if destroy: del self.__tokens[host + ":" + port] else: self.__tokens[host + ":" + port] = token with open(self.token_file, "w") as scfg: portalocker.lock(scfg, portalocker.LOCK_EX) json.dump({"tokens": self.__tokens}, scfg, indent=2, sort_keys=True) portalocker.unlock(scfg)
def increment(self, key, value=1): portalocker.lock(self.locker, portalocker.LOCK_EX) storage = shelve.open(self.shelve_name) try: if storage.has_key(key): value = storage[key][1] + value storage[key] = (time.time(), value) storage.sync() except BaseException, e: portalocker.unlock(self.locker) raise e
def clear(self, regex=None): locker = open(self.locker_name,'a') portalocker.lock(locker, portalocker.LOCK_EX) storage = shelve.open(self.shelve_name) if regex == None: storage.clear() else: self._clear(storage, regex) storage.sync() portalocker.unlock(locker) locker.close()
def file_lock(fd, ltype): if(sys.platform == 'win32'): if(ltype == LOCK): lock(fd, LOCK_EX) elif(ltype == UNLOCK): unlock(fd) else: if(ltype == LOCK): flock(fd, LOCK_EX) elif(ltype == UNLOCK): flock(fd, LOCK_UN)
def save_token(self, host, port, token, destroy=False): if destroy: self.__tokens.pop('{0}:{1}'.format(host, port), None) else: self.__tokens['{0}:{1}'.format(host, port)] = token with open(self.token_file, 'w') as scfg: portalocker.lock(scfg, portalocker.LOCK_EX) json.dump({'tokens': self.__tokens}, scfg, indent=2, sort_keys=True) portalocker.unlock(scfg)
def release(self): """ Release file and thread locks. """ try: if self.stream_lock and not self.stream_lock.closed: unlock(self.stream_lock) except Exception: pass finally: # release thread lock BaseRotatingHandler.release(self)
from time import sleep import requests from subprocess import Popen import portalocker from logzero import logger # start the server if not already started lockfile = f'{Path(__file__).parent.parent / "deepl_fastapi" / "deepl_server.py.portalocker.lock"}' logger.info("lockfile: %s", lockfile) file = open(lockfile, "r+") try: portalocker.lock(file, portalocker.LOCK_EX | portalocker.LOCK_NB) locked = False portalocker.unlock(file) except Exception: locked = True logger.debug("locked: %s", locked) if not locked: cwd = Path(__file__).absolute().parent.as_posix() executable = f"{sys.executable}" if os.name in ["posix"]: # linux and friends cmd = f"nohup python -m deepl_fastapi.run_uvicorn > {cwd}" "/server.out 2>&1 &" Popen(cmd, shell=True) logger.info( "fastapi server running in background, output logged to: %s/server.out", cwd, ) else:
def releaseLock(lockHandler): if lockHandler is None: return portalocker.unlock(lockHandler)
class CacheOnDisk(CacheAbstract): """ Disk based cache This is implemented as a shelve object and it is shared by multiple web2py processes (and threads) as long as they share the same filesystem. The file is locked wen accessed. Disk cache provides persistance when web2py is started/stopped but it slower than `CacheInRam` Values stored in disk cache must be pickable. """ def __init__(self, request, folder=None): self.request = request # Lets test if the cache folder exists, if not # we are going to create it folder = folder or os.path.join(request.folder, 'cache') if not os.path.exists(folder): os.mkdir(folder) ### we need this because of a possible bug in shelve that may ### or may not lock self.locker_name = os.path.join(request.folder, 'cache/cache.lock') self.shelve_name = os.path.join(request.folder, 'cache/cache.shelve') locker, locker_locked = None, False try: locker = open(self.locker_name, 'a') portalocker.lock(locker, portalocker.LOCK_EX) locker_locked = True storage = shelve.open(self.shelve_name) if not storage.has_key(CacheAbstract.cache_stats_name): storage[CacheAbstract.cache_stats_name] = { 'hit_total': 0, 'misses': 0, } storage.sync() except ImportError: pass # no module _bsddb, ignoring exception now so it makes a ticket only if used except: logger.error('corrupted file: %s' % self.shelve_name) if locker_locked: portalocker.unlock(locker) if locker: locker.close() def clear(self, regex=None): locker = open(self.locker_name, 'a') portalocker.lock(locker, portalocker.LOCK_EX) storage = shelve.open(self.shelve_name) if regex == None: storage.clear() else: self._clear(storage, regex) if not CacheAbstract.cache_stats_name in storage.keys(): storage[CacheAbstract.cache_stats_name] = { 'hit_total': 0, 'misses': 0, } storage.sync() portalocker.unlock(locker) locker.close() def __call__(self, key, f, time_expire=DEFAULT_TIME_EXPIRE): dt = time_expire locker = open(self.locker_name, 'a') portalocker.lock(locker, portalocker.LOCK_EX) storage = shelve.open(self.shelve_name) item = storage.get(key, None) if item and f == None: del storage[key] storage[CacheAbstract.cache_stats_name] = { 'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'] + 1, 'misses': storage[CacheAbstract.cache_stats_name]['misses'] } storage.sync() portalocker.unlock(locker) locker.close() if f is None: return None if item and (dt == None or item[0] > time.time() - dt): return item[1] value = f() locker = open(self.locker_name, 'a') portalocker.lock(locker, portalocker.LOCK_EX) storage[key] = (time.time(), value) storage[CacheAbstract.cache_stats_name] = { 'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'], 'misses': storage[CacheAbstract.cache_stats_name]['misses'] + 1 } storage.sync() portalocker.unlock(locker) locker.close() return value def increment(self, key, value=1): locker = open(self.locker_name, 'a') portalocker.lock(locker, portalocker.LOCK_EX) storage = shelve.open(self.shelve_name) try: if key in storage: value = storage[key][1] + value storage[key] = (time.time(), value) storage.sync() except BaseException, e: portalocker.unlock(locker) locker.close() raise e portalocker.unlock(locker) locker.close() return value
def __init__(self, cfunc, arg_types, ctx, cpp_info): idcode = get_func_idcode(cfunc.func_name, arg_types) if ctx not in CTX_FUNC_MAP: CTX_FUNC_MAP[ctx] = dict() cpp_fname = cpp_info.cpp_fname if cpp_fname not in CTX_FUNC_MAP[ctx]: CTX_FUNC_MAP[ctx][cpp_fname] = dict() # func_map: dict mapping idcode to CFunction func_map = CTX_FUNC_MAP[ctx][cpp_fname] if idcode not in func_map: ''' *load function* when one of the following conditions is True: 1. idcode is not loaded 2. loading the function with same function name but different cpp filename ''' cpp_path, cpp_basename = os.path.split(cpp_fname) build_path = os.path.join(cpp_path, 'build') use_template = bool(cfunc.template_list) makedirs(build_path, exist_ok=True) build_info_fname = os.path.join( build_path, os.path.splitext(cpp_basename)[0] + '.json') build_info_fs = open(build_info_fname, 'a+') portalocker.lock(build_info_fs, portalocker.LOCK_EX) build_info_fs.seek(0) js_data = build_info_fs.read() if js_data: map_data = json.loads(js_data) else: map_data = dict(version=OP_LOAD_MODULE_BUILD_VERSION) del js_data # try to load the instance of template function # map_data is a dict which records build information if map_data.get('version') > OP_LOAD_MODULE_BUILD_VERSION: portalocker.unlock(build_info_fs) raise Exception( """Unsupported higher version %s of wrapper file (Current MobulaOP ver: %s) :-(. Please update MobulaOP.""" % (map_data.get('version'), OP_LOAD_MODULE_BUILD_VERSION)) build_id = map_data.get('build_id', 0) is_old_version = map_data.get( 'version') < OP_LOAD_MODULE_BUILD_VERSION # load the information of template functions ORDINARY_FUNCTION_NAME = 'ordinary_functions' TEMPLATE_FUNCTION_NAME = 'template_functions' if is_old_version: ordinary_functions = list() template_functions = dict() else: ordinary_functions = map_data.get(ORDINARY_FUNCTION_NAME, list()) template_functions = map_data.get(TEMPLATE_FUNCTION_NAME, dict()) so_prefix = os.path.join(cpp_path, 'build', os.path.splitext(cpp_basename)[0]) # The filename of build target dll_fname_format = '{prefix}_{ctx}'.format( prefix=so_prefix, ctx=ctx) + '_{build_id}.so' dll_fname = dll_fname_format.format(build_id=build_id) file_changed = file_is_changed(cpp_fname) dll_existed = os.path.exists(dll_fname) func_existed = idcode in template_functions or idcode in ordinary_functions if file_changed or not dll_existed or not func_existed or is_old_version: # Rebuild DLL file try: # try to remove old DLL file os.remove(dll_fname) except: pass if file_changed: # clear template_functions since some functions may have been deleted or renamed after codefile is changed. template_functions.clear() if file_changed or not func_existed: ''' we increase `build_id` by 1 when one of the following conditions is True: 1. the cpp file has been changed 2. new idcode When the cpp file is not changed, and idcode exists in template_functions, `build_id` will be not changed. ''' build_id += 1 dll_fname = dll_fname_format.format(build_id=build_id) # build code code_buffer = _generate_ordinary_code(cpp_info) ordinary_functions = _get_ordinary_functions(cpp_info) if use_template: if idcode not in template_functions: _update_template_inst_map(idcode, template_functions, cfunc, arg_types) # add template instances code into code_buffer code_buffer += ''.join(template_functions.values()) with build_context(): try: _build_lib(cpp_fname, code_buffer, ctx, dll_fname) except: # if build fail, unlock the build info file portalocker.unlock(build_info_fs) raise # update template_functions map_data = dict(version=OP_LOAD_MODULE_BUILD_VERSION, build_id=build_id) map_data[ORDINARY_FUNCTION_NAME] = ordinary_functions map_data[TEMPLATE_FUNCTION_NAME] = template_functions # clear the old context and write json data build_info_fs.seek(0) build_info_fs.truncate() json.dump(map_data, build_info_fs) build_info_fs.flush() os.fsync(build_info_fs.fileno()) portalocker.unlock(build_info_fs) # load all functions in the dll cpp_info.load_dll(dll_fname) # import all functions # ordinary functions for func_name, ord_cfunc in cpp_info.function_args.items(): if not ord_cfunc.template_list: func_idcode = get_func_idcode(func_name, ord_cfunc.arg_types) _add_function(func_map, func_idcode, cpp_info, dll_fname) # template functions for func_idcode in template_functions.keys(): _add_function(func_map, func_idcode, cpp_info, dll_fname) self.func = func_map[idcode].func self.cpp_info = func_map[idcode].cpp_info self.idcode_hash = get_idcode_hash(idcode)
def main(): try: print("10000 entries found for scraping") entries_start = input("Enter Start Number: ") entries = input("Enter Number of entries you want to scrap: ") # excludes = ['DJ', 'repost', 'management', 'group', 'agency', 'singer', 'beat', 'producer', 'prod by'] print("Data Scrapping...") entry_count = int(entries) / 10 if entry_count < 1: entry_count = 1 rapper_data = [] filename = "SoundCloudRapperData.csv" if not os.path.exists(filename): count = 0 else: count = 1 global driver, PROXY, excludes print("PROXY", PROXY) proxies = { "http": "http://" + PROXY, "https": "http://" + PROXY } with open(filename, 'a') as file: portalocker.lock(file, portalocker.LOCK_EX) for n in range(round(entry_count)): url = initialurl.replace('<offset>', str((n * 10) + int(entries_start))) try: data = requests.get(url, proxies=proxies) except: data = requests.get(url) dataStr = data.content.decode('utf-8') dataObj = json.loads(dataStr) # print(dataObj) # print(driver) total_results = dataObj['total_results'] for i in range(len(dataObj['collection'])): print(i) permalink = dataObj['collection'][i]['permalink'] try: driver.get("https://soundcloud.com/" + permalink + "/tracks") except: driver, PROXY, excludes = get_chromedriver() driver.get("https://soundcloud.com/" + permalink + "/tracks") bio = sleep_and_find('//*[@id="content"]/div/div[4]/div[2]/div/article[1]') try: show_more = findByXPATH('//*[@id="content"]/div/div[4]/div[2]/div/article[1]/div[1]/div/a') show_more.click() except: pass try: bio = bio.text except: bio = '' pass excluded_word = False # if re.search(r'\brapper\b', bio, re.IGNORECASE): for exclude in excludes: if re.search(exclude, bio, re.IGNORECASE): excluded_word = True break if not excluded_word: email = re.search(r'\S+@\S+\.\S+', bio, re.IGNORECASE) IG_username, IG_URL = None, None if re.search(r'\binstagram\b', bio, re.IGNORECASE): web_profiles = sleep_and_find( '//*[@id="content"]/div/div[4]/div[2]/div/article[1]/div[2]/ul') if web_profiles: web_profiles_list = web_profiles.find_elements_by_tag_name('li') for i in range(len(web_profiles_list)): href = sleep_and_find( '//*[@id="content"]/div/div[4]/div[2]/div/article[1]/div[2]/ul/li[' + str( (i + 1)) + ']/div/a').get_attribute('href') if 'instagram' in href and href is not None: # IG_username = re.search(r'%2F(.*?)%2F', href) IG_username = href.split(".com%2F")[1].split('&')[0] if not IG_username.find("%2F") == -1: IG_username = IG_username.split("%2F")[0] IG_URL = "https://www.instagram.com/" + IG_username if re.search(r'\bIG: @\b', bio, re.IGNORECASE): web_profiles = sleep_and_find( '//*[@id="content"]/div/div[4]/div[2]/div/article[1]/div[1]/div/div/div/div/p[5]/a[2]') if web_profiles: IG_username = web_profiles.text IG_URL = "https://www.instagram.com/" + IG_username if email is not None or IG_username is not None: username = dataObj['collection'][i]['username'] permalink_url = dataObj['collection'][i]['permalink_url'] full_name = dataObj['collection'][i]['full_name'] country = str(dataObj['collection'][i]['country_code']) if email: try: email = email.group(0) except Exception as ex: email = email.group(1) print(ex) cn = '' if country: cn = country if country else '' location = str(dataObj['collection'][i]['city']) location_sl = '' if not location.find("/") == -1: location_sl = sleep_and_find( '//*[@id="content"]/div/div[2]/div/div[1]/div/div[2]/h4[2]') location_sl = location_sl.text try: location_sl = str(location_sl).split(", ")[1] except: location_sl = '' if location_sl == '': location = location_sl + " " + cn else: location = location + " " + cn name_title = sleep_and_find( '//*[@id="content"]/div/div[4]/div[1]/div/div[2]/div/ul/li[1]/div/div/div[2]/div[1]/div/div/div[2]/a/span') try: name_title = name_title.text except: name_title = '' full_name2 = replace_all(full_name) try: full_name2 = full_name2.split('(')[0] except: pass try: full_name2 = full_name2.split('[')[0] except: pass try: full_name2 = full_name2.split('|')[0] except: pass if '- ' in name_title: rapper_name = name_title.split('-')[0] rapper_name = replace_all(rapper_name) try: rapper_name = rapper_name.split('x ')[0] except: pass try: rapper_name = rapper_name.split(',')[0] except: pass try: rapper_name = rapper_name.split('feat')[0] except: pass try: rapper_name = rapper_name.split('Feat')[0] except: pass try: rapper_name = rapper_name.split('prod')[0] except: pass try: rapper_name = rapper_name.split('(')[0] except: pass try: rapper_name = rapper_name.split('and')[0] except: pass try: rapper_name = rapper_name.split('&')[0] except: pass try: rapper_name = rapper_name.split('+')[0] except: pass try: rapper_name = rapper_name.split('[')[0] except: pass try: rapper_name = rapper_name.split('|')[0] except: pass try: song_title = name_title.split('-')[1] + name_title.split('-')[2] except: song_title = name_title.split('-')[1] song_title_full = song_title try: song_title = song_title.split('(')[0] except: pass try: song_title = song_title.split('[')[0] except: pass try: song_title = song_title.split('feat')[0] except: pass try: song_title = song_title.split('Feat')[0] except: pass else: rapper_name = replace_all(username) try: rapper_name = rapper_name.split('(')[0] except: pass try: rapper_name = rapper_name.split('[')[0] except: pass try: rapper_name = rapper_name.split('|')[0] except: pass song_title = name_title song_title_full = name_title try: song_title = name_title.split('(')[0] except: pass try: song_title = song_title.split('[')[0] except: pass try: song_title = song_title.split('Feat')[0] except: pass try: song_title = song_title.split('feat')[0] except: pass try: song_title = song_title.split('ft')[0] except: pass try: song_title = song_title.split('Ft')[0] except: pass if full_name2 == '': full_name2 = rapper_name.strip() song_title = song_title.strip() song_title = song_title.encode("ascii", "ignore") song_title = song_title.decode() full_name2 = full_name2.encode("ascii", "ignore") full_name2 = full_name2.decode() if email: if not email.find(":") == -1: email = email.split(":")[1] if not email.find("/") == -1: email = email.split("//")[1] rapper_list = [username, rapper_name.strip(), full_name2, location, country, permalink_url, email, song_title, song_title_full, IG_URL, IG_username] rapper_data.append(rapper_list) if len(rapper_data) > 0: count += 1 portalocker.unlock(file) rapper_data_df = pd.DataFrame(rapper_data) rapper_data_df.columns = ['Username', 'FullName', 'FullName2', 'Location', 'Country', 'RapperURL', 'Email', 'SongTitle', 'SongTitleFull', 'InstagramURL', 'InstagramUserName'] if count == 1: rapper_data_df.to_csv(filename, mode='a', index=False, header=True) else: rapper_data_df.to_csv(filename, mode='a', index=False, header=False) print(str((n + 1) * 100) + " Entries Scrapped Successfully and saved into " + filename) rapper_data = [] else: print("No Rapper Found in 100 entries") driver, PROXY, excludes = get_chromedriver() print('All data Scrapped Successfully and saved in File ' + filename) k = input("******* press any key to exit *******") except Exception as ex: print(ex, 'Please first close the file.') k = input("******* press any key to exit *******")
def unlock(self): if self.locked: portalocker.unlock(self._f) self.locked = False
file_handle = open(cmd[1], 'r+') except IOError: sys.stderr.write("Error attempting to lock " + cmd[1]) print "" else: lock_list[cmd[1]] = Locker(LOCK_TYPE, file_handle) if LOCK_TYPE == "EX": portalocker.lock(file_handle, portalocker.LOCK_EX) elif LOCK_TYPE == "SH": portalocker.lock(file_handle, portalocker.LOCK_SH) elif LOCK_TYPE == "NB": portalocker.lock(file_handle, portalocker.LOCK_NB) print "Locked" elif cmd[0] in ['unlock', 'u']: if cmd[1] in lock_list.keys(): portalocker.unlock(lock_list[cmd[1]].get_fh()) print "Unlocked" del (lock_list[cmd[1]]) else: print cmd[1] + " is not locked by locker" elif cmd[0] in ['type', 't']: try: cmd[1] except IndexError: print "Lock Type is " + LOCK_TYPE else: if cmd[1].upper() == "EX" or cmd[1].upper( ) == "SH" or cmd[1].upper() == "NB": LOCK_TYPE = cmd[1].upper() print "Lock type set to " + cmd[1].upper() else:
portalocker.lock(locker, portalocker.LOCK_EX) locker_locked = True storage = shelve.open(self.shelve_name) if not storage.has_key(CacheAbstract.cache_stats_name): storage[CacheAbstract.cache_stats_name] = { 'hit_total': 0, 'misses': 0, } storage.sync() except ImportError, e: pass # no module _bsddb, ignoring exception now so it makes a ticket only if used except: logging.error('corrupted file: %s' % self.shelve_name) if locker_locked: portalocker.unlock(locker) if locker: locker.close() def clear(self, regex=None): locker = open(self.locker_name, 'a') portalocker.lock(locker, portalocker.LOCK_EX) storage = shelve.open(self.shelve_name) if regex == None: storage.clear() else: self._clear(storage, regex) if not CacheAbstract.cache_stats_name in storage.keys(): storage[CacheAbstract.cache_stats_name] = { 'hit_total': 0, 'misses': 0,
def __exit__(self, exc_type, exc_val, exc_tb): portalocker.unlock(self._lockfile) self._lockfile.close()
def address_to_long_lat(city, state, addressline1, addressline2, postcode, country): """ :param city: e.g. :param state: e.g. :param addressline1: e.g. :param addressline2: e.g. :param postcode: e.g. :param country: e.g. :return: """ with open(SCRIPT_DIR / '..' / 'data' / 'geolocate_cache.json', 'r', encoding='utf-8') as f: portalocker.lock(f, portalocker.LOCK_SH) try: address_cache = json.loads(f.read()) finally: portalocker.unlock(f) address = [] if addressline2: address.append(addressline2) if addressline1: address.append(addressline1) if city: address.append(city) if state: address.append(state) if postcode: address.append(postcode) if country: address.append(country) address = ', '.join(address) if address in address_cache: item = address_cache[address] if item or True: return item if not GEOLOCATOR[0]: GEOLOCATOR[0] = MapBox(api_key=environ['MAPBOX_KEY'], user_agent='https://www.covid-19-au.com') try: location = GEOLOCATOR[0].geocode(address) if location: longlat = (location.longitude, location.latitude) except: # Make sure it doesn't keep trying to get the lat/long # when the service didn't find the location! traceback.print_exc() longlat = None time.sleep(1) address_cache[address] = longlat with open(SCRIPT_DIR / '..' / 'data' / 'geolocate_cache.json', 'w', encoding='utf-8') as f: portalocker.lock(f, portalocker.LOCK_EX) try: f.write(json.dumps(address_cache, ensure_ascii=False, indent=2)) finally: portalocker.unlock(f) return longlat
def unlock(self): self._locked = False self._file.flush() portalocker.unlock(self._file)
def unlock(self): """Unlock the file if it is currently locked""" if self.locked: self._f.flush() portalocker.unlock(self._f) self.locked = False
def __init__(self): LOG.debug("Loading clientside session config.") # Check whether user's configuration exists. session_cfg_file = get_password_file() LOG.info("Checking local passwords or tokens in %s", session_cfg_file) scfg_dict = {} user_home = os.path.expanduser("~") mistyped_cfg_file = os.path.join(user_home, ".codechecker.password.json") if os.path.exists(session_cfg_file): check_file_owner_rw(session_cfg_file) scfg_dict = load_json_or_empty(session_cfg_file, {}, "user authentication") scfg_dict['credentials'] = \ simplify_credentials(scfg_dict['credentials']) if not scfg_dict['credentials']: LOG.info("No saved tokens.") else: LOG.debug("Tokens or passwords were found for these hosts:") for k, v in scfg_dict['credentials'].items(): user, _ = v.split(":") LOG.debug(" user '%s' host '%s'", user, k) elif os.path.exists(mistyped_cfg_file): LOG.warning("Typo in file name! Rename '%s' to '%s'.", mistyped_cfg_file, session_cfg_file) else: LOG.info("Password file not found.") if not scfg_dict.get('credentials'): scfg_dict['credentials'] = {} self.__save = scfg_dict self.__autologin = scfg_dict.get('client_autologin', True) # Check and load token storage for user. self.token_file = get_session_file() LOG.info("Checking for local valid sessions.") with open(self.token_file, 'a+', encoding="utf-8", errors="ignore") as f: f.seek(0) try: portalocker.lock(f, portalocker.LOCK_EX) token_dict = json.loads(f.read()) check_file_owner_rw(self.token_file) self.__tokens = token_dict.get('tokens', {}) LOG.debug("Found session information for these hosts:") for k, _ in self.__tokens.items(): LOG.debug(" %s", k) except json.JSONDecodeError: json.dump({'tokens': {}}, f) os.chmod(self.token_file, stat.S_IRUSR | stat.S_IWUSR) self.__tokens = {} finally: portalocker.unlock(f)
def write_to_file(self): f = open('result.tmp', 'a+') portalocker.lock(f, portalocker.LOCK_UN) f.write(str(self.node_id) + ':believed\n') portalocker.unlock(f) f.close()
# Recreates dict with keys starting from 0 counter = 0 number_of_new_leads = len(data) array = {} for key, value in data.iteritems(): #print key turn_into_string2 = str(counter) array[turn_into_string2] = value #print value counter += 1 #total_number_of_items_in_data_master_json # Export file new_lead.json with open("new_lead.json", 'w') as outfile: json.dump(array, outfile) exit # Append the new leads to the master.json file add_to_end = total_number_of_items_in_data_master_json for key, value in data.iteritems(): turn_into_string3 = str(add_to_end) master[turn_into_string3] = value add_to_end += 1 master_json.seek(0) json.dump(master, master_json) master_json.truncate() portalocker.unlock(master_json)
def unlock(self): """if file is locked, then unlock it""" if self.locked: self._f.flush() portalocker.unlock(self._f) self.locked = False
def unlock(self): if self.locked: self._f.flush() portalocker.unlock(self._f) self.locked = False
def souncloudscrapper(use_proxy, use_manual_proxy): driver = None try: print("10000 entries found for scraping") entries_start = input("Enter Start Number: ") entries = input("Enter Number of entries you want to scrap: ") excludes = get_excludes() print("Data Scrapping...") entry_count = int(entries) / 100 if entry_count < 1: entry_count = 1 rapper_data_email = [] rapper_data_instagram = [] filenameEmail = "SoundCloudRapperDataEmail.csv" filenameInstagram = "SoundCloudRapperDataInstagram.csv" pCounter = 1 if not os.path.exists(filenameEmail): countEmail = 0 else: countEmail = 1 if not os.path.exists(filenameEmail): countInsta = 0 else: countInsta = 1 instaFile = open(filenameInstagram, 'a') emailFile = open(filenameEmail, 'a') portalocker.lock(emailFile, portalocker.LOCK_EX) portalocker.lock(instaFile, portalocker.LOCK_EX) for n in range(round(entry_count)): url = initialurl.replace('<offset>', str((n * 100) + int(entries_start))) if use_proxy: proxies = get_proxies() print(proxies) for i in range(25): s, p = get_session(proxies) try: data = s.get(url, timeout=12) dataStr = data.content.decode('utf-8') dataObj = json.loads(dataStr) rapper_data_email, rapper_data_instagram, e = proxy_changer( 30, dataObj, excludes, rapper_data_email, rapper_data_instagram, proxies) if e: print("No data found using proxies Try Again.") exit(0) except Exception as e: print("Max time is exceeded Switching to other proxy.") continue if i == 24: print("No data found using auto proxies Try Again.") exit(0) elif use_manual_proxy: pCounter += 1 driver, proxy, total_proxies = get_chromedriver() proxies = { "http": "http://" + proxy, "https": "http://" + proxy } try: data = requests.get(url, proxies=proxies) except: print( "Max time is exceeded Switching to other manual proxy." ) continue if pCounter == total_proxies: print("No data found using manual proxies Try Again.") exit(0) dataStr = data.content.decode('utf-8') dataObj = json.loads(dataStr) for j in range(len(dataObj['collection'])): permalink = dataObj['collection'][j]['permalink'] driver.get("https://soundcloud.com/" + permalink + "/tracks") rapper_data_email, rapper_data_instagram = main_scrapper_function( dataObj, excludes, rapper_data_email, rapper_data_instagram, driver, j) else: data = requests.get(url) dataStr = data.content.decode('utf-8') dataObj = json.loads(dataStr) driver = webdriver.Chrome(options=options, executable_path=DRIVER_PATH, service_log_path="NULL") for j in range(len(dataObj['collection'])): permalink = dataObj['collection'][j]['permalink'] try: driver.set_page_load_timeout(10) driver.get("https://soundcloud.com/" + permalink + "/tracks") rapper_data_email, rapper_data_instagram = main_scrapper_function( dataObj, excludes, rapper_data_email, rapper_data_instagram, driver, j) except Exception as ex: print(ex) total_results = dataObj['total_results'] if len(rapper_data_email) > 0: countEmail += 1 portalocker.unlock(filenameEmail) rapper_data_df = pd.DataFrame(rapper_data_email) rapper_data_df.columns = [ 'Username', 'FullName', 'FullName2', 'Location', 'Country', 'RapperURL', 'Email', 'SongTitle', 'SongTitleFull', 'InstagramURL', 'InstagramUserName' ] if countEmail == 1: rapper_data_df.to_csv(filenameEmail, mode='a', index=False, header=True) else: rapper_data_df.to_csv(filenameEmail, mode='a', index=False, header=False) print( str((n + 1) * 100) + " Entries Scrapped Successfully and saved into " + filenameEmail) rapper_data_email = [] else: print("No Rapper Found in 100 entries") if len(rapper_data_instagram) > 0: countEmail += 1 portalocker.unlock(filenameInstagram) rapper_data_df = pd.DataFrame(rapper_data_instagram) rapper_data_df.columns = [ 'Username', 'FullName', 'FullName2', 'Location', 'Country', 'RapperURL', 'InstagramURL', 'InstagramUserName', 'SongTitle', 'SongTitleFull' ] if countInsta == 1: rapper_data_df.to_csv(filenameInstagram, mode='a', index=False, header=True) else: rapper_data_df.to_csv(filenameInstagram, mode='a', index=False, header=False) print( str((n + 1) * 100) + " Entries Scrapped Successfully and saved into " + filenameInstagram) rapper_data_instagram = [] else: print("No Rapper Found in 100 entries") if not rapper_data_email or rapper_data_instagram: print("No Rapper Found") else: print('All data Scrapped Successfully and saved in Files') k = input("******* press any key to exit ******* \t") if k: exit(1) except Exception as ex: print(ex, 'Please first close the file.') k = input("******* press any key to exit *******") if driver is not None: driver.quit()
def release_lock(file): '''Release our lock on the given path ''' portalocker.unlock(file)
def unlock(self): if self.locked: # Flush internal buffers and operating system buffers self._f.flush() portalocker.unlock(self._f) self.locked = False
def unlock(): portalocker.unlock(f) f.close()
def unlock(): for f in locked_files: portalocker.unlock(f) locked_files.remove(f)
def _unlock(self, response): if response and response.session_file: try: portalocker.unlock(response.session_file) except: ### this should never happen but happens in Windows pass
def _close(self): try: shelve.Shelf.close(self) finally: portalocker.unlock(self.locker) self.locker.close()
import random import time import portalocker """ Process 3 constantly tries to aquire the lock and holds it for a random period of time to confuse p2. file_name: the file that p1, p2 use to transmit data zero_time: length p1 holding the lock to signal a 0 one_time: length p1 holding the lock to signal a 1 """ file_name = 'empty.txt' zero_time = 2 one_time = 8 while True: time.sleep(random.random() * 20 * zero_time) # it controls the frequency of interferences try: with open(file_name, 'w') as f: portalocker.lock(f, portalocker.LOCK_EX) time.sleep(random.random() * (zero_time + one_time)) portalocker.unlock(f) except: pass
def __exit__(self, type, value, traceback): portalocker.unlock(self._f)
def update_description_file(self, dimension): """Create the description file with a JSON structure :param: dimension 2-tuple, the dimension of the image frame (w, h) """ dest_path = self.qc_params.root_folder html_path = os.path.join(dest_path, 'index.html') # Make sure the file exists before trying to open it in 'r+' mode open(html_path, 'a').close() # NB: We use 'r+' because it allows us to open an existing file for locking *without* immediately truncating the # existing contents prior to opening. We can then use this file to overwrite the contents later in the function. dest_file = open(html_path, 'r+', encoding="utf-8") # We lock `index.html` at the start of this function so that we halt any other processes *before* they have a # chance to generate or read any .json files. This ensues that the last process to write to index.html has read # in all of the available .json files, preventing: # https://github.com/spinalcordtoolbox/spinalcordtoolbox/pull/3701#discussion_r816300380 portalocker.lock(dest_file, portalocker.LOCK_EX) output = { 'python': sys.executable, 'cwd': self.qc_params.cwd, 'cmdline': "{} {}".format(self.qc_params.command, self.qc_params.args), 'command': self.qc_params.command, 'sct_version': self.qc_params.sct_version, 'dataset': self.qc_params.dataset, 'subject': self.qc_params.subject, 'contrast': self.qc_params.contrast, 'fname_in': self.qc_params.fname_in, 'orientation': self.qc_params.orientation, 'background_img': self.qc_params.bkg_img_path, 'overlay_img': self.qc_params.overlay_img_path, 'dimension': '%dx%d' % dimension, 'moddate': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 'qc': "" } logger.debug('Description file: %s', self.qc_params.qc_results) # results = [] # Create path to store json files path_json, _ = os.path.split(self.qc_params.qc_results) if not os.path.exists(path_json): os.makedirs(path_json, exist_ok=True) # Create json file for specific QC entry with open(self.qc_params.qc_results, 'w+') as qc_file: json.dump(output, qc_file, indent=1) # Append entry to existing HTML file json_data = get_json_data_from_path(path_json) assets_path = os.path.join(os.path.dirname(__file__), 'assets') with open(os.path.join(assets_path, 'index.html'), encoding="utf-8") as template_index: template = Template(template_index.read()) output = template.substitute(sct_json_data=json.dumps(json_data)) # Empty the HTML file before writing, to make sure there's no leftover junk at the end dest_file.truncate() dest_file.write(output) for path in ['css', 'js', 'imgs', 'fonts']: src_path = os.path.join(assets_path, '_assets', path) dest_full_path = os.path.join(dest_path, '_assets', path) if not os.path.exists(dest_full_path): os.makedirs(dest_full_path, exist_ok=True) for file_ in os.listdir(src_path): if not os.path.isfile(os.path.join(dest_full_path, file_)): copy(os.path.join(src_path, file_), dest_full_path) dest_file.flush() portalocker.unlock(dest_file) dest_file.close()
def save_storage(storage, filename): fp = open(filename, 'wb') portalocker.lock(fp, portalocker.LOCK_EX) cPickle.dump(dict(storage), fp) portalocker.unlock(fp) fp.close()
def unlock_root(self): self._file.flush() portalocker.unlock(self._root_lock_file)
import os import sys import main_window from portalocker import lock, LOCK_EX, unlock from PyQt5.QtWidgets import QMessageBox from PyQt5.QtWidgets import QApplication lockfile_name = 'pyformula.lck' if __name__ == '__main__': app = QApplication(sys.argv) win = main_window.MainWindow() if os.path.exists(lockfile_name): QMessageBox.warning(win, "Warning", \ ''' You have opened a window of PyFormula. If not, please delete the \'pyformula.lck\' file. ''') else: with open(lockfile_name, 'w') as lockfile: lock(lockfile, LOCK_EX) win.show() return_code = app.exec() unlock(lockfile) os.remove(lockfile_name) sys.exit(return_code)