def _unlock(self, response):
if response and response.session_file and response.session_locked:
try:
portalocker.unlock(response.session_file)
response.session_locked = False
except: # this should never happen but happens in Windows
pass
def findT(path, language='en-us'):
"""
must be run by the admin app
"""
filename = os.path.join(path, 'languages', '%s.py' % language)
sentences = read_dict(filename)
mp = os.path.join(path, 'models')
cp = os.path.join(path, 'controllers')
vp = os.path.join(path, 'views')
for file in listdir(mp, '.+\.py', 0) + listdir(cp, '.+\.py', 0)\
+ listdir(vp, '.+\.html', 0):
fp = open(file, 'r')
portalocker.lock(fp, portalocker.LOCK_SH)
data = fp.read()
portalocker.unlock(fp)
fp.close()
items = regex_translate.findall(data)
for item in items:
try:
msg = eval(item)
if msg and not msg in sentences:
sentences[msg] = msg
except:
pass
write_dict(filename, sentences)
def planet():
#return ""
import gluon.contrib.rss2 as rss2
# store planet rss entries in disk (forever...)
import portalocker
import os, cPickle as pickle
path = os.path.join(request.folder,'cache', "planet.rss")
if not os.path.exists(path):
f = open(path, "w+")
rss = get_planet_rss(None)
rss = [{'title': item.title, 'author': item.author, 'pubDate': item.pubDate, 'link': item.link, 'description': item.description} for item in rss.items]
else:
f = open(path, "r+")
rss = None
portalocker.lock(f, portalocker.LOCK_EX)
if not rss:
rss = pickle.load(f)
else:
f.seek(0)
pickle.dump(rss, f)
portalocker.unlock(f)
f.close()
# .rss requests
if request.extension == "rss":
# return new rss feed xml
response.headers['Content-Type']='application/rss+xml'
return rss2.dumps(rss)
# else send the rss object to be processed by
# the view
return response.render(dict(rss = rss, rss2 = rss2))
def test_simple():
fh = open('tests/test_file.txt', 'r+')
portalocker.lock(fh, portalocker.LOCK_EX)
fh.seek(12)
fh.write('foo')
portalocker.unlock(fh)
fh.close()
def unlock(self,key): try: portalocker.unlock(self.locked[key]) except: print "Cache error unlocking file with key " + key try: self.locked[key].close() except: print "Cache error closing file with key " + key del self.open[key] del self.locked[key]
def __getCachedData(self,key): fname = self.__fname(key) # Pre-existing locked read if(self.locked.get(key)): self.locked[key].seek(0) try: str = pickle.load(self.locked[key]) except: str = False self.locked[key].seek(0) return str fp=open(fname, "r") self.open[key] = fp portalocker.lock(fp,portalocker.LOCK_SH) # The following 2 lines handle cases where open (above) was called # on an empty file that was created by cache::lock() fp.seek(0) try: str = pickle.load(fp) except: str = False try: portalocker.unlock(fp) except: print "Cache error unlocking file with key " + key try: fp.close() except: print "Cache error closing file with key " + key del self.open[key] return str
def __init__(self, request, folder=None):
self.request = request
# Lets test if the cache folder exists, if not
# we are going to create it
folder = folder or os.path.join(request.folder, "cache")
if not os.path.exists(folder):
os.mkdir(folder)
### we need this because of a possible bug in shelve that may
### or may not lock
self.locker_name = os.path.join(request.folder, "cache/cache.lock")
self.shelve_name = os.path.join(request.folder, "cache/cache.shelve")
locker, locker_locked = None, False
try:
locker = open(self.locker_name, "a")
portalocker.lock(locker, portalocker.LOCK_EX)
locker_locked = True
storage = shelve.open(self.shelve_name)
if not storage.has_key(CacheAbstract.cache_stats_name):
storage[CacheAbstract.cache_stats_name] = {"hit_total": 0, "misses": 0}
storage.sync()
except ImportError:
pass # no module _bsddb, ignoring exception now so it makes a ticket only if used
except:
logger.error("corrupted file: %s" % self.shelve_name)
if locker_locked:
portalocker.unlock(locker)
if locker:
locker.close()
def from_conf(cls, path=None, **overrides):
'''Initialize instance from YAML configuration file,
writing updates (only to keys, specified by "conf_update_keys") back to it.'''
import yaml
if path is None:
path = cls.conf_path_default
log.debug('Using default state-file path: {}'.format(path))
path = os.path.expanduser(path)
with open(path, 'r') as src:
portalocker.lock(src, portalocker.LOCK_SH)
# fcntl.lockf(src, fcntl.LOCK_SH)
conf = yaml.load(src.read())
portalocker.unlock(src)
conf.setdefault('conf_save', path)
conf_cls = dict()
for ns, keys in cls.conf_update_keys.viewitems():
for k in keys:
try:
v = conf.get(ns, dict()).get(k)
except AttributeError:
if not cls.conf_raise_structure_errors: raise
raise KeyError('Unable to get value for configuration parameter'
' "{k}" in section "{ns}", check configuration file (path: {path}) syntax'
' near the aforementioned section/value.'.format(ns=ns, k=k, path=path))
if v is not None:
conf_cls['{}_{}'.format(ns, k)] = conf[ns][k]
conf_cls.update(overrides)
self = cls(**conf_cls)
self.conf_save = conf['conf_save']
return self
def findT(path, language='en-us'):
"""
must be run by the admin app
"""
filename = os.path.join(path, 'languages', '%s.py' % language)
sentences = read_dict(filename)
mp = os.path.join(path, 'models')
cp = os.path.join(path, 'controllers')
vp = os.path.join(path, 'views')
for file in listdir(mp, '.+\.py', 0) + listdir(cp, '.+\.py', 0)\
+ listdir(vp, '.+\.html', 0):
fp = open(file, 'r')
portalocker.lock(fp, portalocker.LOCK_SH)
data = fp.read()
portalocker.unlock(fp)
fp.close()
items = regex_translate.findall(data)
for item in items:
try:
message = eval(item)
if not message.startswith('#') and not '\n' in message:
tokens = message.rsplit('##', 1)
else:
# this allows markmin syntax in translations
tokens = [message]
if len(tokens) == 2:
message = tokens[0].strip() + '##' + tokens[1].strip()
if message and not message in sentences:
sentences[message] = message
except:
pass
write_dict(filename, sentences)
def findT(path, language="en-us"):
"""
must be run by the admin app
"""
filename = os.path.join(path, "languages", "%s.py" % language)
sentences = read_dict(filename)
mp = os.path.join(path, "models")
cp = os.path.join(path, "controllers")
vp = os.path.join(path, "views")
for file in listdir(mp, ".+\.py", 0) + listdir(cp, ".+\.py", 0) + listdir(vp, ".+\.html", 0):
fp = open(file, "r")
portalocker.lock(fp, portalocker.LOCK_SH)
data = fp.read()
portalocker.unlock(fp)
fp.close()
items = regex_translate.findall(data)
for item in items:
try:
message = eval(item)
if not message.startswith("#") and not "\n" in message:
tokens = message.rsplit("##", 1)
else:
# this allows markmin syntax in translations
tokens = [message]
if len(tokens) == 2:
message = tokens[0].strip() + "##" + tokens[1].strip()
if message and not message in sentences:
sentences[message] = message
except:
pass
write_dict(filename, sentences)
def _open_shelve_and_lock(self):
"""Open and return a shelf object, obtaining an exclusive lock
on self.locker first. Replaces the close method of the
returned shelf instance with one that releases the lock upon
closing."""
storage = None
locker = None
locked = False
try:
locker = locker = open(self.locker_name, 'a')
portalocker.lock(locker, portalocker.LOCK_EX)
locked = True
try:
storage = shelve.open(self.shelve_name)
except:
logger.error('corrupted cache file %s, will try rebuild it' \
% (self.shelve_name))
storage = None
if not storage and os.path.exists(self.shelve_name):
os.unlink(self.shelve_name)
storage = shelve.open(self.shelve_name)
if not CacheAbstract.cache_stats_name in storage.keys():
storage[CacheAbstract.cache_stats_name] = {'hit_total':0, 'misses': 0}
storage.sync()
except Exception, e:
if storage:
storage.close()
storage = None
if locked:
portalocker.unlock(locker)
locker.close()
locked = False
raise RuntimeError, 'unable to create/re-create cache file %s' % self.shelve_name
def load_storage(filename):
fp = open(filename, 'rb')
portalocker.lock(fp, portalocker.LOCK_EX)
storage = cPickle.load(fp)
portalocker.unlock(fp)
fp.close()
return Storage(storage)
def myunlock(myfilename):
lock_file = portalock_open(myfilename)
lock_file.truncate()
lock_file.write('clear\n')
lock_file.flush()
portalocker.unlock(lock_file)
lock_file.close()
def __call__(
self,
key,
f,
time_expire = DEFAULT_TIME_EXPIRE,
):
dt = time_expire
locker = open(self.locker_name,'a')
portalocker.lock(locker, portalocker.LOCK_EX)
storage = shelve.open(self.shelve_name)
item = storage.get(key, None)
if item and f == None:
del storage[key]
portalocker.unlock(locker)
locker.close()
if f is None:
return None
if item and (dt == None or item[0] > time.time() - dt):
return item[1]
value = f()
locker = open(self.locker_name,'a')
portalocker.lock(locker, portalocker.LOCK_EX)
storage[key] = (time.time(), value)
storage.sync()
portalocker.unlock(locker)
locker.close()
return value
def progress_bar(current, left):
with open('progress_bar.lock', 'w') as lockfile:
portalocker.lock(lockfile, portalocker.LOCK_EX)
progress = 120 * (current/left)
stringbuilder = '[{0}] {1}%'.format('#' * (progress / 12), progress * 120 * 100)
lockfile.write(stringbuilder)
say(stringbuilder)
portalocker.unlock(lockfile)
def _unlock(self):
response = current.response
if response and getattr(response,'session_file',None) and getattr(response,'session_locked',None):
try:
portalocker.unlock(response.session_file)
response.session_locked = False
except: # this should never happen but happens in Windows
pass
def save_storage(storage, filename):
fp = open(filename, 'wb')
try:
portalocker.lock(fp, portalocker.LOCK_EX)
cPickle.dump(dict(storage), fp)
portalocker.unlock(fp)
finally:
fp.close()
def _unlock(self, response):
if response and response.session_file:
try:
portalocker.unlock(response.session_file)
response.session_file.close()
del response.session_file
except: ### this should never happen but happens in Windows
pass
def write_dict(filename, contents):
fp = open(filename, "w")
portalocker.lock(fp, portalocker.LOCK_EX)
fp.write("# coding: utf8\n{\n")
for key in sorted(contents):
fp.write("%s: %s,\n" % (repr(key), repr(contents[key])))
fp.write("}\n")
portalocker.unlock(fp)
fp.close()
def write_dict(filename, contents):
fp = open(filename, 'w')
portalocker.lock(fp, portalocker.LOCK_EX)
fp.write('{\n')
for key in sorted(contents):
fp.write('%s: %s,\n' % (repr(key), repr(contents[key])))
fp.write('}\n')
portalocker.unlock(fp)
fp.close()
def _close_shelve_and_unlock(self):
try:
if self.storage:
self.storage.close()
finally:
if self.locker and self.locked:
portalocker.unlock(self.locker)
self.locker.close()
self.locked = False
def release(self):
try:
self.stream.flush()
finally:
try:
unlock(self.stream_lock)
finally:
# release thread lock
Handler.release(self)
def read_dict(filename):
fp = open(filename, 'r')
portalocker.lock(fp, portalocker.LOCK_SH)
lang_text = fp.read().replace('\r\n', '\n')
portalocker.unlock(fp)
fp.close()
if not lang_text.strip():
return {}
return eval(lang_text)
def func():
start = time.time()
with file('a.txt', 'w') as f:
lock(f, LOCK_SH)
time.sleep(0.2)
f.seek(0,2)
f.write(str(os.getpid()) + ', ' + str(time.time()) + '\n')
unlock(f)
end = time.time()
print start,end, start-end
def saveToken(self, host, port, token, destroy=False):
if destroy:
del self.__tokens[host + ":" + port]
else:
self.__tokens[host + ":" + port] = token
with open(self.token_file, "w") as scfg:
portalocker.lock(scfg, portalocker.LOCK_EX)
json.dump({"tokens": self.__tokens}, scfg, indent=2, sort_keys=True)
portalocker.unlock(scfg)
def increment(self, key, value=1):
portalocker.lock(self.locker, portalocker.LOCK_EX)
storage = shelve.open(self.shelve_name)
try:
if storage.has_key(key):
value = storage[key][1] + value
storage[key] = (time.time(), value)
storage.sync()
except BaseException, e:
portalocker.unlock(self.locker)
raise e
def clear(self, regex=None):
locker = open(self.locker_name,'a')
portalocker.lock(locker, portalocker.LOCK_EX)
storage = shelve.open(self.shelve_name)
if regex == None:
storage.clear()
else:
self._clear(storage, regex)
storage.sync()
portalocker.unlock(locker)
locker.close()
def file_lock(fd, ltype):
if(sys.platform == 'win32'):
if(ltype == LOCK):
lock(fd, LOCK_EX)
elif(ltype == UNLOCK):
unlock(fd)
else:
if(ltype == LOCK):
flock(fd, LOCK_EX)
elif(ltype == UNLOCK):
flock(fd, LOCK_UN)
def save_token(self, host, port, token, destroy=False):
if destroy:
self.__tokens.pop('{0}:{1}'.format(host, port), None)
else:
self.__tokens['{0}:{1}'.format(host, port)] = token
with open(self.token_file, 'w') as scfg:
portalocker.lock(scfg, portalocker.LOCK_EX)
json.dump({'tokens': self.__tokens}, scfg,
indent=2, sort_keys=True)
portalocker.unlock(scfg)
def release(self):
""" Release file and thread locks.
"""
try:
if self.stream_lock and not self.stream_lock.closed:
unlock(self.stream_lock)
except Exception:
pass
finally:
# release thread lock
BaseRotatingHandler.release(self)
from time import sleep
import requests
from subprocess import Popen
import portalocker
from logzero import logger
# start the server if not already started
lockfile = f'{Path(__file__).parent.parent / "deepl_fastapi" / "deepl_server.py.portalocker.lock"}'
logger.info("lockfile: %s", lockfile)
file = open(lockfile, "r+")
try:
portalocker.lock(file, portalocker.LOCK_EX | portalocker.LOCK_NB)
locked = False
portalocker.unlock(file)
except Exception:
locked = True
logger.debug("locked: %s", locked)
if not locked:
cwd = Path(__file__).absolute().parent.as_posix()
executable = f"{sys.executable}"
if os.name in ["posix"]: # linux and friends
cmd = f"nohup python -m deepl_fastapi.run_uvicorn > {cwd}" "/server.out 2>&1 &"
Popen(cmd, shell=True)
logger.info(
"fastapi server running in background, output logged to: %s/server.out",
cwd,
)
else:
def releaseLock(lockHandler):
if lockHandler is None: return
portalocker.unlock(lockHandler)
class CacheOnDisk(CacheAbstract):
"""
Disk based cache
This is implemented as a shelve object and it is shared by multiple web2py
processes (and threads) as long as they share the same filesystem.
The file is locked wen accessed.
Disk cache provides persistance when web2py is started/stopped but it slower
than `CacheInRam`
Values stored in disk cache must be pickable.
"""
def __init__(self, request, folder=None):
self.request = request
# Lets test if the cache folder exists, if not
# we are going to create it
folder = folder or os.path.join(request.folder, 'cache')
if not os.path.exists(folder):
os.mkdir(folder)
### we need this because of a possible bug in shelve that may
### or may not lock
self.locker_name = os.path.join(request.folder, 'cache/cache.lock')
self.shelve_name = os.path.join(request.folder, 'cache/cache.shelve')
locker, locker_locked = None, False
try:
locker = open(self.locker_name, 'a')
portalocker.lock(locker, portalocker.LOCK_EX)
locker_locked = True
storage = shelve.open(self.shelve_name)
if not storage.has_key(CacheAbstract.cache_stats_name):
storage[CacheAbstract.cache_stats_name] = {
'hit_total': 0,
'misses': 0,
}
storage.sync()
except ImportError:
pass # no module _bsddb, ignoring exception now so it makes a ticket only if used
except:
logger.error('corrupted file: %s' % self.shelve_name)
if locker_locked:
portalocker.unlock(locker)
if locker:
locker.close()
def clear(self, regex=None):
locker = open(self.locker_name, 'a')
portalocker.lock(locker, portalocker.LOCK_EX)
storage = shelve.open(self.shelve_name)
if regex == None:
storage.clear()
else:
self._clear(storage, regex)
if not CacheAbstract.cache_stats_name in storage.keys():
storage[CacheAbstract.cache_stats_name] = {
'hit_total': 0,
'misses': 0,
}
storage.sync()
portalocker.unlock(locker)
locker.close()
def __call__(self, key, f, time_expire=DEFAULT_TIME_EXPIRE):
dt = time_expire
locker = open(self.locker_name, 'a')
portalocker.lock(locker, portalocker.LOCK_EX)
storage = shelve.open(self.shelve_name)
item = storage.get(key, None)
if item and f == None:
del storage[key]
storage[CacheAbstract.cache_stats_name] = {
'hit_total':
storage[CacheAbstract.cache_stats_name]['hit_total'] + 1,
'misses': storage[CacheAbstract.cache_stats_name]['misses']
}
storage.sync()
portalocker.unlock(locker)
locker.close()
if f is None:
return None
if item and (dt == None or item[0] > time.time() - dt):
return item[1]
value = f()
locker = open(self.locker_name, 'a')
portalocker.lock(locker, portalocker.LOCK_EX)
storage[key] = (time.time(), value)
storage[CacheAbstract.cache_stats_name] = {
'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'],
'misses': storage[CacheAbstract.cache_stats_name]['misses'] + 1
}
storage.sync()
portalocker.unlock(locker)
locker.close()
return value
def increment(self, key, value=1):
locker = open(self.locker_name, 'a')
portalocker.lock(locker, portalocker.LOCK_EX)
storage = shelve.open(self.shelve_name)
try:
if key in storage:
value = storage[key][1] + value
storage[key] = (time.time(), value)
storage.sync()
except BaseException, e:
portalocker.unlock(locker)
locker.close()
raise e
portalocker.unlock(locker)
locker.close()
return value
def __init__(self, cfunc, arg_types, ctx, cpp_info):
idcode = get_func_idcode(cfunc.func_name, arg_types)
if ctx not in CTX_FUNC_MAP:
CTX_FUNC_MAP[ctx] = dict()
cpp_fname = cpp_info.cpp_fname
if cpp_fname not in CTX_FUNC_MAP[ctx]:
CTX_FUNC_MAP[ctx][cpp_fname] = dict()
# func_map: dict mapping idcode to CFunction
func_map = CTX_FUNC_MAP[ctx][cpp_fname]
if idcode not in func_map:
'''
*load function* when one of the following conditions is True:
1. idcode is not loaded
2. loading the function with same function name but different cpp filename
'''
cpp_path, cpp_basename = os.path.split(cpp_fname)
build_path = os.path.join(cpp_path, 'build')
use_template = bool(cfunc.template_list)
makedirs(build_path, exist_ok=True)
build_info_fname = os.path.join(
build_path,
os.path.splitext(cpp_basename)[0] + '.json')
build_info_fs = open(build_info_fname, 'a+')
portalocker.lock(build_info_fs, portalocker.LOCK_EX)
build_info_fs.seek(0)
js_data = build_info_fs.read()
if js_data:
map_data = json.loads(js_data)
else:
map_data = dict(version=OP_LOAD_MODULE_BUILD_VERSION)
del js_data
# try to load the instance of template function
# map_data is a dict which records build information
if map_data.get('version') > OP_LOAD_MODULE_BUILD_VERSION:
portalocker.unlock(build_info_fs)
raise Exception(
"""Unsupported higher version %s of wrapper file (Current MobulaOP ver: %s) :-(.
Please update MobulaOP.""" %
(map_data.get('version'), OP_LOAD_MODULE_BUILD_VERSION))
build_id = map_data.get('build_id', 0)
is_old_version = map_data.get(
'version') < OP_LOAD_MODULE_BUILD_VERSION
# load the information of template functions
ORDINARY_FUNCTION_NAME = 'ordinary_functions'
TEMPLATE_FUNCTION_NAME = 'template_functions'
if is_old_version:
ordinary_functions = list()
template_functions = dict()
else:
ordinary_functions = map_data.get(ORDINARY_FUNCTION_NAME,
list())
template_functions = map_data.get(TEMPLATE_FUNCTION_NAME,
dict())
so_prefix = os.path.join(cpp_path, 'build',
os.path.splitext(cpp_basename)[0])
# The filename of build target
dll_fname_format = '{prefix}_{ctx}'.format(
prefix=so_prefix, ctx=ctx) + '_{build_id}.so'
dll_fname = dll_fname_format.format(build_id=build_id)
file_changed = file_is_changed(cpp_fname)
dll_existed = os.path.exists(dll_fname)
func_existed = idcode in template_functions or idcode in ordinary_functions
if file_changed or not dll_existed or not func_existed or is_old_version:
# Rebuild DLL file
try:
# try to remove old DLL file
os.remove(dll_fname)
except:
pass
if file_changed:
# clear template_functions since some functions may have been deleted or renamed after codefile is changed.
template_functions.clear()
if file_changed or not func_existed:
'''
we increase `build_id` by 1 when one of the following conditions is True:
1. the cpp file has been changed
2. new idcode
When the cpp file is not changed, and idcode exists in template_functions,
`build_id` will be not changed.
'''
build_id += 1
dll_fname = dll_fname_format.format(build_id=build_id)
# build code
code_buffer = _generate_ordinary_code(cpp_info)
ordinary_functions = _get_ordinary_functions(cpp_info)
if use_template:
if idcode not in template_functions:
_update_template_inst_map(idcode, template_functions,
cfunc, arg_types)
# add template instances code into code_buffer
code_buffer += ''.join(template_functions.values())
with build_context():
try:
_build_lib(cpp_fname, code_buffer, ctx, dll_fname)
except:
# if build fail, unlock the build info file
portalocker.unlock(build_info_fs)
raise
# update template_functions
map_data = dict(version=OP_LOAD_MODULE_BUILD_VERSION,
build_id=build_id)
map_data[ORDINARY_FUNCTION_NAME] = ordinary_functions
map_data[TEMPLATE_FUNCTION_NAME] = template_functions
# clear the old context and write json data
build_info_fs.seek(0)
build_info_fs.truncate()
json.dump(map_data, build_info_fs)
build_info_fs.flush()
os.fsync(build_info_fs.fileno())
portalocker.unlock(build_info_fs)
# load all functions in the dll
cpp_info.load_dll(dll_fname)
# import all functions
# ordinary functions
for func_name, ord_cfunc in cpp_info.function_args.items():
if not ord_cfunc.template_list:
func_idcode = get_func_idcode(func_name,
ord_cfunc.arg_types)
_add_function(func_map, func_idcode, cpp_info, dll_fname)
# template functions
for func_idcode in template_functions.keys():
_add_function(func_map, func_idcode, cpp_info, dll_fname)
self.func = func_map[idcode].func
self.cpp_info = func_map[idcode].cpp_info
self.idcode_hash = get_idcode_hash(idcode)
def main():
try:
print("10000 entries found for scraping")
entries_start = input("Enter Start Number: ")
entries = input("Enter Number of entries you want to scrap: ")
# excludes = ['DJ', 'repost', 'management', 'group', 'agency', 'singer', 'beat', 'producer', 'prod by']
print("Data Scrapping...")
entry_count = int(entries) / 10
if entry_count < 1:
entry_count = 1
rapper_data = []
filename = "SoundCloudRapperData.csv"
if not os.path.exists(filename):
count = 0
else:
count = 1
global driver, PROXY, excludes
print("PROXY", PROXY)
proxies = {
"http": "http://" + PROXY,
"https": "http://" + PROXY
}
with open(filename, 'a') as file:
portalocker.lock(file, portalocker.LOCK_EX)
for n in range(round(entry_count)):
url = initialurl.replace('<offset>', str((n * 10) + int(entries_start)))
try:
data = requests.get(url, proxies=proxies)
except:
data = requests.get(url)
dataStr = data.content.decode('utf-8')
dataObj = json.loads(dataStr)
# print(dataObj)
# print(driver)
total_results = dataObj['total_results']
for i in range(len(dataObj['collection'])):
print(i)
permalink = dataObj['collection'][i]['permalink']
try:
driver.get("https://soundcloud.com/" + permalink + "/tracks")
except:
driver, PROXY, excludes = get_chromedriver()
driver.get("https://soundcloud.com/" + permalink + "/tracks")
bio = sleep_and_find('//*[@id="content"]/div/div[4]/div[2]/div/article[1]')
try:
show_more = findByXPATH('//*[@id="content"]/div/div[4]/div[2]/div/article[1]/div[1]/div/a')
show_more.click()
except:
pass
try:
bio = bio.text
except:
bio = ''
pass
excluded_word = False
# if re.search(r'\brapper\b', bio, re.IGNORECASE):
for exclude in excludes:
if re.search(exclude, bio, re.IGNORECASE):
excluded_word = True
break
if not excluded_word:
email = re.search(r'\S+@\S+\.\S+', bio, re.IGNORECASE)
IG_username, IG_URL = None, None
if re.search(r'\binstagram\b', bio, re.IGNORECASE):
web_profiles = sleep_and_find(
'//*[@id="content"]/div/div[4]/div[2]/div/article[1]/div[2]/ul')
if web_profiles:
web_profiles_list = web_profiles.find_elements_by_tag_name('li')
for i in range(len(web_profiles_list)):
href = sleep_and_find(
'//*[@id="content"]/div/div[4]/div[2]/div/article[1]/div[2]/ul/li[' + str(
(i + 1)) + ']/div/a').get_attribute('href')
if 'instagram' in href and href is not None:
# IG_username = re.search(r'%2F(.*?)%2F', href)
IG_username = href.split(".com%2F")[1].split('&')[0]
if not IG_username.find("%2F") == -1:
IG_username = IG_username.split("%2F")[0]
IG_URL = "https://www.instagram.com/" + IG_username
if re.search(r'\bIG: @\b', bio, re.IGNORECASE):
web_profiles = sleep_and_find(
'//*[@id="content"]/div/div[4]/div[2]/div/article[1]/div[1]/div/div/div/div/p[5]/a[2]')
if web_profiles:
IG_username = web_profiles.text
IG_URL = "https://www.instagram.com/" + IG_username
if email is not None or IG_username is not None:
username = dataObj['collection'][i]['username']
permalink_url = dataObj['collection'][i]['permalink_url']
full_name = dataObj['collection'][i]['full_name']
country = str(dataObj['collection'][i]['country_code'])
if email:
try:
email = email.group(0)
except Exception as ex:
email = email.group(1)
print(ex)
cn = ''
if country:
cn = country if country else ''
location = str(dataObj['collection'][i]['city'])
location_sl = ''
if not location.find("/") == -1:
location_sl = sleep_and_find(
'//*[@id="content"]/div/div[2]/div/div[1]/div/div[2]/h4[2]')
location_sl = location_sl.text
try:
location_sl = str(location_sl).split(", ")[1]
except:
location_sl = ''
if location_sl == '':
location = location_sl + " " + cn
else:
location = location + " " + cn
name_title = sleep_and_find(
'//*[@id="content"]/div/div[4]/div[1]/div/div[2]/div/ul/li[1]/div/div/div[2]/div[1]/div/div/div[2]/a/span')
try:
name_title = name_title.text
except:
name_title = ''
full_name2 = replace_all(full_name)
try:
full_name2 = full_name2.split('(')[0]
except:
pass
try:
full_name2 = full_name2.split('[')[0]
except:
pass
try:
full_name2 = full_name2.split('|')[0]
except:
pass
if '- ' in name_title:
rapper_name = name_title.split('-')[0]
rapper_name = replace_all(rapper_name)
try:
rapper_name = rapper_name.split('x ')[0]
except:
pass
try:
rapper_name = rapper_name.split(',')[0]
except:
pass
try:
rapper_name = rapper_name.split('feat')[0]
except:
pass
try:
rapper_name = rapper_name.split('Feat')[0]
except:
pass
try:
rapper_name = rapper_name.split('prod')[0]
except:
pass
try:
rapper_name = rapper_name.split('(')[0]
except:
pass
try:
rapper_name = rapper_name.split('and')[0]
except:
pass
try:
rapper_name = rapper_name.split('&')[0]
except:
pass
try:
rapper_name = rapper_name.split('+')[0]
except:
pass
try:
rapper_name = rapper_name.split('[')[0]
except:
pass
try:
rapper_name = rapper_name.split('|')[0]
except:
pass
try:
song_title = name_title.split('-')[1] + name_title.split('-')[2]
except:
song_title = name_title.split('-')[1]
song_title_full = song_title
try:
song_title = song_title.split('(')[0]
except:
pass
try:
song_title = song_title.split('[')[0]
except:
pass
try:
song_title = song_title.split('feat')[0]
except:
pass
try:
song_title = song_title.split('Feat')[0]
except:
pass
else:
rapper_name = replace_all(username)
try:
rapper_name = rapper_name.split('(')[0]
except:
pass
try:
rapper_name = rapper_name.split('[')[0]
except:
pass
try:
rapper_name = rapper_name.split('|')[0]
except:
pass
song_title = name_title
song_title_full = name_title
try:
song_title = name_title.split('(')[0]
except:
pass
try:
song_title = song_title.split('[')[0]
except:
pass
try:
song_title = song_title.split('Feat')[0]
except:
pass
try:
song_title = song_title.split('feat')[0]
except:
pass
try:
song_title = song_title.split('ft')[0]
except:
pass
try:
song_title = song_title.split('Ft')[0]
except:
pass
if full_name2 == '':
full_name2 = rapper_name.strip()
song_title = song_title.strip()
song_title = song_title.encode("ascii", "ignore")
song_title = song_title.decode()
full_name2 = full_name2.encode("ascii", "ignore")
full_name2 = full_name2.decode()
if email:
if not email.find(":") == -1:
email = email.split(":")[1]
if not email.find("/") == -1:
email = email.split("//")[1]
rapper_list = [username, rapper_name.strip(), full_name2, location, country, permalink_url,
email, song_title, song_title_full, IG_URL, IG_username]
rapper_data.append(rapper_list)
if len(rapper_data) > 0:
count += 1
portalocker.unlock(file)
rapper_data_df = pd.DataFrame(rapper_data)
rapper_data_df.columns = ['Username', 'FullName', 'FullName2', 'Location', 'Country', 'RapperURL',
'Email', 'SongTitle', 'SongTitleFull', 'InstagramURL',
'InstagramUserName']
if count == 1:
rapper_data_df.to_csv(filename, mode='a', index=False, header=True)
else:
rapper_data_df.to_csv(filename, mode='a', index=False, header=False)
print(str((n + 1) * 100) + " Entries Scrapped Successfully and saved into " + filename)
rapper_data = []
else:
print("No Rapper Found in 100 entries")
driver, PROXY, excludes = get_chromedriver()
print('All data Scrapped Successfully and saved in File ' + filename)
k = input("******* press any key to exit *******")
except Exception as ex:
print(ex, 'Please first close the file.')
k = input("******* press any key to exit *******")
def unlock(self):
if self.locked:
portalocker.unlock(self._f)
self.locked = False
file_handle = open(cmd[1], 'r+')
except IOError:
sys.stderr.write("Error attempting to lock " + cmd[1])
print ""
else:
lock_list[cmd[1]] = Locker(LOCK_TYPE, file_handle)
if LOCK_TYPE == "EX":
portalocker.lock(file_handle, portalocker.LOCK_EX)
elif LOCK_TYPE == "SH":
portalocker.lock(file_handle, portalocker.LOCK_SH)
elif LOCK_TYPE == "NB":
portalocker.lock(file_handle, portalocker.LOCK_NB)
print "Locked"
elif cmd[0] in ['unlock', 'u']:
if cmd[1] in lock_list.keys():
portalocker.unlock(lock_list[cmd[1]].get_fh())
print "Unlocked"
del (lock_list[cmd[1]])
else:
print cmd[1] + " is not locked by locker"
elif cmd[0] in ['type', 't']:
try:
cmd[1]
except IndexError:
print "Lock Type is " + LOCK_TYPE
else:
if cmd[1].upper() == "EX" or cmd[1].upper(
) == "SH" or cmd[1].upper() == "NB":
LOCK_TYPE = cmd[1].upper()
print "Lock type set to " + cmd[1].upper()
else:
portalocker.lock(locker, portalocker.LOCK_EX)
locker_locked = True
storage = shelve.open(self.shelve_name)
if not storage.has_key(CacheAbstract.cache_stats_name):
storage[CacheAbstract.cache_stats_name] = {
'hit_total': 0,
'misses': 0,
}
storage.sync()
except ImportError, e:
pass # no module _bsddb, ignoring exception now so it makes a ticket only if used
except:
logging.error('corrupted file: %s' % self.shelve_name)
if locker_locked:
portalocker.unlock(locker)
if locker:
locker.close()
def clear(self, regex=None):
locker = open(self.locker_name, 'a')
portalocker.lock(locker, portalocker.LOCK_EX)
storage = shelve.open(self.shelve_name)
if regex == None:
storage.clear()
else:
self._clear(storage, regex)
if not CacheAbstract.cache_stats_name in storage.keys():
storage[CacheAbstract.cache_stats_name] = {
'hit_total': 0,
'misses': 0,
def __exit__(self, exc_type, exc_val, exc_tb):
portalocker.unlock(self._lockfile)
self._lockfile.close()
def address_to_long_lat(city, state, addressline1, addressline2, postcode,
country):
"""
:param city: e.g.
:param state: e.g.
:param addressline1: e.g.
:param addressline2: e.g.
:param postcode: e.g.
:param country: e.g.
:return:
"""
with open(SCRIPT_DIR / '..' / 'data' / 'geolocate_cache.json',
'r',
encoding='utf-8') as f:
portalocker.lock(f, portalocker.LOCK_SH)
try:
address_cache = json.loads(f.read())
finally:
portalocker.unlock(f)
address = []
if addressline2:
address.append(addressline2)
if addressline1:
address.append(addressline1)
if city:
address.append(city)
if state:
address.append(state)
if postcode:
address.append(postcode)
if country:
address.append(country)
address = ', '.join(address)
if address in address_cache:
item = address_cache[address]
if item or True:
return item
if not GEOLOCATOR[0]:
GEOLOCATOR[0] = MapBox(api_key=environ['MAPBOX_KEY'],
user_agent='https://www.covid-19-au.com')
try:
location = GEOLOCATOR[0].geocode(address)
if location:
longlat = (location.longitude, location.latitude)
except:
# Make sure it doesn't keep trying to get the lat/long
# when the service didn't find the location!
traceback.print_exc()
longlat = None
time.sleep(1)
address_cache[address] = longlat
with open(SCRIPT_DIR / '..' / 'data' / 'geolocate_cache.json',
'w',
encoding='utf-8') as f:
portalocker.lock(f, portalocker.LOCK_EX)
try:
f.write(json.dumps(address_cache, ensure_ascii=False, indent=2))
finally:
portalocker.unlock(f)
return longlat
def unlock(self):
self._locked = False
self._file.flush()
portalocker.unlock(self._file)
def unlock(self):
"""Unlock the file if it is currently locked"""
if self.locked:
self._f.flush()
portalocker.unlock(self._f)
self.locked = False
def __init__(self):
LOG.debug("Loading clientside session config.")
# Check whether user's configuration exists.
session_cfg_file = get_password_file()
LOG.info("Checking local passwords or tokens in %s", session_cfg_file)
scfg_dict = {}
user_home = os.path.expanduser("~")
mistyped_cfg_file = os.path.join(user_home,
".codechecker.password.json")
if os.path.exists(session_cfg_file):
check_file_owner_rw(session_cfg_file)
scfg_dict = load_json_or_empty(session_cfg_file, {},
"user authentication")
scfg_dict['credentials'] = \
simplify_credentials(scfg_dict['credentials'])
if not scfg_dict['credentials']:
LOG.info("No saved tokens.")
else:
LOG.debug("Tokens or passwords were found for these hosts:")
for k, v in scfg_dict['credentials'].items():
user, _ = v.split(":")
LOG.debug(" user '%s' host '%s'", user, k)
elif os.path.exists(mistyped_cfg_file):
LOG.warning("Typo in file name! Rename '%s' to '%s'.",
mistyped_cfg_file, session_cfg_file)
else:
LOG.info("Password file not found.")
if not scfg_dict.get('credentials'):
scfg_dict['credentials'] = {}
self.__save = scfg_dict
self.__autologin = scfg_dict.get('client_autologin', True)
# Check and load token storage for user.
self.token_file = get_session_file()
LOG.info("Checking for local valid sessions.")
with open(self.token_file, 'a+', encoding="utf-8",
errors="ignore") as f:
f.seek(0)
try:
portalocker.lock(f, portalocker.LOCK_EX)
token_dict = json.loads(f.read())
check_file_owner_rw(self.token_file)
self.__tokens = token_dict.get('tokens', {})
LOG.debug("Found session information for these hosts:")
for k, _ in self.__tokens.items():
LOG.debug(" %s", k)
except json.JSONDecodeError:
json.dump({'tokens': {}}, f)
os.chmod(self.token_file, stat.S_IRUSR | stat.S_IWUSR)
self.__tokens = {}
finally:
portalocker.unlock(f)
def write_to_file(self):
f = open('result.tmp', 'a+')
portalocker.lock(f, portalocker.LOCK_UN)
f.write(str(self.node_id) + ':believed\n')
portalocker.unlock(f)
f.close()
# Recreates dict with keys starting from 0
counter = 0
number_of_new_leads = len(data)
array = {}
for key, value in data.iteritems():
#print key
turn_into_string2 = str(counter)
array[turn_into_string2] = value
#print value
counter += 1
#total_number_of_items_in_data_master_json
# Export file new_lead.json
with open("new_lead.json", 'w') as outfile:
json.dump(array, outfile)
exit
# Append the new leads to the master.json file
add_to_end = total_number_of_items_in_data_master_json
for key, value in data.iteritems():
turn_into_string3 = str(add_to_end)
master[turn_into_string3] = value
add_to_end += 1
master_json.seek(0)
json.dump(master, master_json)
master_json.truncate()
portalocker.unlock(master_json)
def unlock(self):
"""if file is locked, then unlock it"""
if self.locked:
self._f.flush()
portalocker.unlock(self._f)
self.locked = False
def unlock(self):
if self.locked:
self._f.flush()
portalocker.unlock(self._f)
self.locked = False
def souncloudscrapper(use_proxy, use_manual_proxy):
driver = None
try:
print("10000 entries found for scraping")
entries_start = input("Enter Start Number: ")
entries = input("Enter Number of entries you want to scrap: ")
excludes = get_excludes()
print("Data Scrapping...")
entry_count = int(entries) / 100
if entry_count < 1:
entry_count = 1
rapper_data_email = []
rapper_data_instagram = []
filenameEmail = "SoundCloudRapperDataEmail.csv"
filenameInstagram = "SoundCloudRapperDataInstagram.csv"
pCounter = 1
if not os.path.exists(filenameEmail):
countEmail = 0
else:
countEmail = 1
if not os.path.exists(filenameEmail):
countInsta = 0
else:
countInsta = 1
instaFile = open(filenameInstagram, 'a')
emailFile = open(filenameEmail, 'a')
portalocker.lock(emailFile, portalocker.LOCK_EX)
portalocker.lock(instaFile, portalocker.LOCK_EX)
for n in range(round(entry_count)):
url = initialurl.replace('<offset>',
str((n * 100) + int(entries_start)))
if use_proxy:
proxies = get_proxies()
print(proxies)
for i in range(25):
s, p = get_session(proxies)
try:
data = s.get(url, timeout=12)
dataStr = data.content.decode('utf-8')
dataObj = json.loads(dataStr)
rapper_data_email, rapper_data_instagram, e = proxy_changer(
30, dataObj, excludes, rapper_data_email,
rapper_data_instagram, proxies)
if e:
print("No data found using proxies Try Again.")
exit(0)
except Exception as e:
print("Max time is exceeded Switching to other proxy.")
continue
if i == 24:
print("No data found using auto proxies Try Again.")
exit(0)
elif use_manual_proxy:
pCounter += 1
driver, proxy, total_proxies = get_chromedriver()
proxies = {
"http": "http://" + proxy,
"https": "http://" + proxy
}
try:
data = requests.get(url, proxies=proxies)
except:
print(
"Max time is exceeded Switching to other manual proxy."
)
continue
if pCounter == total_proxies:
print("No data found using manual proxies Try Again.")
exit(0)
dataStr = data.content.decode('utf-8')
dataObj = json.loads(dataStr)
for j in range(len(dataObj['collection'])):
permalink = dataObj['collection'][j]['permalink']
driver.get("https://soundcloud.com/" + permalink +
"/tracks")
rapper_data_email, rapper_data_instagram = main_scrapper_function(
dataObj, excludes, rapper_data_email,
rapper_data_instagram, driver, j)
else:
data = requests.get(url)
dataStr = data.content.decode('utf-8')
dataObj = json.loads(dataStr)
driver = webdriver.Chrome(options=options,
executable_path=DRIVER_PATH,
service_log_path="NULL")
for j in range(len(dataObj['collection'])):
permalink = dataObj['collection'][j]['permalink']
try:
driver.set_page_load_timeout(10)
driver.get("https://soundcloud.com/" + permalink +
"/tracks")
rapper_data_email, rapper_data_instagram = main_scrapper_function(
dataObj, excludes, rapper_data_email,
rapper_data_instagram, driver, j)
except Exception as ex:
print(ex)
total_results = dataObj['total_results']
if len(rapper_data_email) > 0:
countEmail += 1
portalocker.unlock(filenameEmail)
rapper_data_df = pd.DataFrame(rapper_data_email)
rapper_data_df.columns = [
'Username', 'FullName', 'FullName2', 'Location', 'Country',
'RapperURL', 'Email', 'SongTitle', 'SongTitleFull',
'InstagramURL', 'InstagramUserName'
]
if countEmail == 1:
rapper_data_df.to_csv(filenameEmail,
mode='a',
index=False,
header=True)
else:
rapper_data_df.to_csv(filenameEmail,
mode='a',
index=False,
header=False)
print(
str((n + 1) * 100) +
" Entries Scrapped Successfully and saved into " +
filenameEmail)
rapper_data_email = []
else:
print("No Rapper Found in 100 entries")
if len(rapper_data_instagram) > 0:
countEmail += 1
portalocker.unlock(filenameInstagram)
rapper_data_df = pd.DataFrame(rapper_data_instagram)
rapper_data_df.columns = [
'Username', 'FullName', 'FullName2', 'Location', 'Country',
'RapperURL', 'InstagramURL', 'InstagramUserName',
'SongTitle', 'SongTitleFull'
]
if countInsta == 1:
rapper_data_df.to_csv(filenameInstagram,
mode='a',
index=False,
header=True)
else:
rapper_data_df.to_csv(filenameInstagram,
mode='a',
index=False,
header=False)
print(
str((n + 1) * 100) +
" Entries Scrapped Successfully and saved into " +
filenameInstagram)
rapper_data_instagram = []
else:
print("No Rapper Found in 100 entries")
if not rapper_data_email or rapper_data_instagram:
print("No Rapper Found")
else:
print('All data Scrapped Successfully and saved in Files')
k = input("******* press any key to exit ******* \t")
if k:
exit(1)
except Exception as ex:
print(ex, 'Please first close the file.')
k = input("******* press any key to exit *******")
if driver is not None:
driver.quit()
def release_lock(file):
'''Release our lock on the given path
'''
portalocker.unlock(file)
def unlock(self):
if self.locked:
# Flush internal buffers and operating system buffers
self._f.flush()
portalocker.unlock(self._f)
self.locked = False
def unlock():
portalocker.unlock(f)
f.close()
def unlock():
for f in locked_files:
portalocker.unlock(f)
locked_files.remove(f)
def _unlock(self, response):
if response and response.session_file:
try:
portalocker.unlock(response.session_file)
except: ### this should never happen but happens in Windows
pass
def _close(self):
try:
shelve.Shelf.close(self)
finally:
portalocker.unlock(self.locker)
self.locker.close()
import random
import time
import portalocker
"""
Process 3 constantly tries to aquire the lock and holds it for a random period of time to confuse p2.
file_name: the file that p1, p2 use to transmit data
zero_time: length p1 holding the lock to signal a 0
one_time: length p1 holding the lock to signal a 1
"""
file_name = 'empty.txt'
zero_time = 2
one_time = 8
while True:
time.sleep(random.random() * 20 *
zero_time) # it controls the frequency of interferences
try:
with open(file_name, 'w') as f:
portalocker.lock(f, portalocker.LOCK_EX)
time.sleep(random.random() * (zero_time + one_time))
portalocker.unlock(f)
except:
pass
def __exit__(self, type, value, traceback):
portalocker.unlock(self._f)
def update_description_file(self, dimension):
"""Create the description file with a JSON structure
:param: dimension 2-tuple, the dimension of the image frame (w, h)
"""
dest_path = self.qc_params.root_folder
html_path = os.path.join(dest_path, 'index.html')
# Make sure the file exists before trying to open it in 'r+' mode
open(html_path, 'a').close()
# NB: We use 'r+' because it allows us to open an existing file for locking *without* immediately truncating the
# existing contents prior to opening. We can then use this file to overwrite the contents later in the function.
dest_file = open(html_path, 'r+', encoding="utf-8")
# We lock `index.html` at the start of this function so that we halt any other processes *before* they have a
# chance to generate or read any .json files. This ensues that the last process to write to index.html has read
# in all of the available .json files, preventing:
# https://github.com/spinalcordtoolbox/spinalcordtoolbox/pull/3701#discussion_r816300380
portalocker.lock(dest_file, portalocker.LOCK_EX)
output = {
'python': sys.executable,
'cwd': self.qc_params.cwd,
'cmdline': "{} {}".format(self.qc_params.command,
self.qc_params.args),
'command': self.qc_params.command,
'sct_version': self.qc_params.sct_version,
'dataset': self.qc_params.dataset,
'subject': self.qc_params.subject,
'contrast': self.qc_params.contrast,
'fname_in': self.qc_params.fname_in,
'orientation': self.qc_params.orientation,
'background_img': self.qc_params.bkg_img_path,
'overlay_img': self.qc_params.overlay_img_path,
'dimension': '%dx%d' % dimension,
'moddate': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
'qc': ""
}
logger.debug('Description file: %s', self.qc_params.qc_results)
# results = []
# Create path to store json files
path_json, _ = os.path.split(self.qc_params.qc_results)
if not os.path.exists(path_json):
os.makedirs(path_json, exist_ok=True)
# Create json file for specific QC entry
with open(self.qc_params.qc_results, 'w+') as qc_file:
json.dump(output, qc_file, indent=1)
# Append entry to existing HTML file
json_data = get_json_data_from_path(path_json)
assets_path = os.path.join(os.path.dirname(__file__), 'assets')
with open(os.path.join(assets_path, 'index.html'),
encoding="utf-8") as template_index:
template = Template(template_index.read())
output = template.substitute(sct_json_data=json.dumps(json_data))
# Empty the HTML file before writing, to make sure there's no leftover junk at the end
dest_file.truncate()
dest_file.write(output)
for path in ['css', 'js', 'imgs', 'fonts']:
src_path = os.path.join(assets_path, '_assets', path)
dest_full_path = os.path.join(dest_path, '_assets', path)
if not os.path.exists(dest_full_path):
os.makedirs(dest_full_path, exist_ok=True)
for file_ in os.listdir(src_path):
if not os.path.isfile(os.path.join(dest_full_path, file_)):
copy(os.path.join(src_path, file_), dest_full_path)
dest_file.flush()
portalocker.unlock(dest_file)
dest_file.close()
def save_storage(storage, filename):
fp = open(filename, 'wb')
portalocker.lock(fp, portalocker.LOCK_EX)
cPickle.dump(dict(storage), fp)
portalocker.unlock(fp)
fp.close()
def unlock_root(self):
self._file.flush()
portalocker.unlock(self._root_lock_file)
import os
import sys
import main_window
from portalocker import lock, LOCK_EX, unlock
from PyQt5.QtWidgets import QMessageBox
from PyQt5.QtWidgets import QApplication
lockfile_name = 'pyformula.lck'
if __name__ == '__main__':
app = QApplication(sys.argv)
win = main_window.MainWindow()
if os.path.exists(lockfile_name):
QMessageBox.warning(win, "Warning", \
'''
You have opened a window of PyFormula.
If not, please delete the \'pyformula.lck\' file.
''')
else:
with open(lockfile_name, 'w') as lockfile:
lock(lockfile, LOCK_EX)
win.show()
return_code = app.exec()
unlock(lockfile)
os.remove(lockfile_name)
sys.exit(return_code)
