def operate(self):
self.busy = True
print "--ID " + self.processing + " Running FilterInput "
self.output = None
lock_file = FileLock(self.__filename)
status = lock_file.is_locked()
while status:
status = lock_file.is_locked()
if os.path.isfile(self.__filename):
file_open = open(self.__filename, "rb")
self.__plain_data = file_open.read()
file_open.close()
if len(self.__plain_data) < 1:
print "Error 400! File input is empty. [File - " + self.__filename + \
"]\n--Terminating ID " + self.processing + " in FilterInput."
else:
self.next_pipe.add_storage([
self.mode, self.processing,
[self.__plain_data, self.__filename]
])
self.busy = False
def run(self):
self.busy = True
filename = self._filter_input[0]
data = self._filter_input[1]
print "--ID " + self.processing + " Running FilterWrite"
if self.mode == "encode":
output_file = filename + ".d2f"
else:
temp_pos = filename.rfind('.')
output_file = filename[:temp_pos]
lock_file = None
if os.path.isfile(output_file):
lock_file = FileLock(output_file)
status = lock_file.is_locked()
while status:
status = lock_file.is_locked()
lock_file.acquire()
file_open = open(output_file, "wb")
file_open.write(data)
file_open.close()
if lock_file:
lock_file.release()
self.taken = True
print "--ID " + self.processing + " Finish"
self.busy = False
self.output = "Ada isinya"
def run(self):
self.busy = True
key = datetime.now().strftime('%Y%m%d%H%M%S') + str(randint(100, 199))
self.processing = key
print "--ID " + self.processing + " Running FilterInput "
self.output = None
lock_file = FileLock(self.__filename)
status = lock_file.is_locked()
while status:
status = lock_file.is_locked()
file_open = open(self.__filename, "rb")
self.__plain_data = file_open.read()
file_open.close()
if len(self.__plain_data) < 1:
print "Error 400! File input is empty. [File - " + self.__filename + "]\n--Terminating ID " + \
self.processing + " in FilterInput."
self.taken = True
else:
self.output = [self.__plain_data, self.__filename]
self.busy = False
def actionStopFCGI(self):
from lockfile import FileLock
from os import kill
from time import sleep
kill(int(open("wl-fcgi.pid", "r").read()), 15)
lock = FileLock("/tmp/wl-fcgi")
countdown = 15
while lock.is_locked() and countdown > 0:
countdown -= 1
if lock.is_locked():
exit(1)
def break_lock(self):
from lockfile import FileLock
lock = FileLock(self.lock_path)
if lock.is_locked():
lock.break_lock()
def break_lock(self):
from lockfile import FileLock
lock = FileLock(self.lock_path)
if lock.is_locked():
lock.break_lock()
def daemonize(args, callback):
with DaemonContext():
from Pyflix.utils.logger import log_set_up
log_set_up(True)
log = logging.getLogger('pyflix.daemon')
log.info("running daemon")
create_process = False
lock = Lock(LOCKFILE, os.getpid(), args.name, args.sea_ep[0],
args.sea_ep[1], args.port)
if lock.is_locked():
log.debug("lock active")
lock_pid = lock.get_pid()
if not lock.is_same_file(args.name, args.sea_ep[0],
args.sea_ep[1]) \
or not is_process_running(lock_pid):
try:
log.debug("killing process %s" % lock_pid)
os.kill(lock_pid, signal.SIGQUIT)
except OSError:
pass
except TypeError:
pass
lock.break_lock()
create_process = True
else:
create_process = True
if create_process:
log.debug("creating proccess")
lock.acquire()
callback()
lock.release()
else:
log.debug("same daemon process")
def main(argv=None):
global argparser, lockfilepath
global session, server, db
if argv is None:
args = argparser.parse_args()
else:
args = argparser.parse_args(argv)
try:
session, server, db = getCheshire3Env(args)
except (EnvironmentError, ObjectDoesNotExistException):
return 1
with db.get_path(session, 'defaultLogger') as session.logger:
mp = db.get_path(session, 'metadataPath')
lock = FileLock(mp)
if lock.is_locked() and args.unlock:
# Forcibly unlock
session.logger.log_warning(session, "Unlocking Database")
lock.break_lock()
try:
lock.acquire(timeout=30) # wait up to 30 seconds
except LockTimeout:
msg = ("The database is locked. It is possible that another"
"user is currently indexing this database. Please wait at "
"least 10 minutes and then try again. If you continue to "
"get this message and you are sure no one is reindexing "
"the database please contact the archives hub team for "
"advice."
)
session.logger.log_critical(session, msg)
return 1
try:
return args.func(args)
finally:
lock.release()
def run(self):
self.busy = True
print "--ID " + self.processing + " Running FilterDictionary"
filename = self._filter_input[1]
temp_pos = filename.rfind('.')
filename = filename[:temp_pos]
filename += ".d2c"
temp = ""
lock_file = FileLock(filename)
status = lock_file.is_locked()
while status:
status = lock_file.is_locked()
if os.path.isfile(filename):
file_open = open(filename, "rb")
temp = file_open.read()
file_open.close()
temp_pos = temp.find('_')
size = int(temp[:temp_pos])
temp = temp[temp_pos + 1:]
temp_list = temp.split("=*")
dict_binary = {}
for i in range(0, len(temp_list)):
temp = temp_list[i]
temp_list2 = temp.split("/|")
temp_list3 = temp_list2[0].split('-')
temp_binary = "{0:b}".format(int(temp_list3[1]))
temp_binary = temp_binary.replace(" ", "")
temp_length = int(temp_list3[0])
if temp_length < len(temp_binary):
range_binary = len(temp_binary) - temp_length
bin_val = temp_binary[range_binary:]
elif temp_length > len(temp_binary):
range_binary = temp_length - len(temp_binary)
bin_val = "0" * range_binary
bin_val += temp_binary
else:
bin_val = temp_binary
dict_binary[bin_val] = temp_list2[1]
self.output = [filename, size, dict_binary, self._filter_input[0]]
self.busy = False
def run(self):
self.busy = True
key = datetime.now().strftime('%Y%m%d%H%M%S') + str(randint(100, 199))
self.processing = key
print "--ID " + self.processing + " Running FilterInput "
self.output = None
lock_file = FileLock(self.__filename)
status = lock_file.is_locked()
while status:
status = lock_file.is_locked()
if os.path.isfile(self.__filename):
file_open = open(self.__filename, "rb")
self.__plain_data = file_open.read()
file_open.close()
self.output = [self.__plain_data, self.__filename]
self.busy = False
def run_locked(self):
lock = FileLock('/tmp/send_notifications')
try:
logger.debug('Trying to acquire lock')
lock.acquire(3)
self.run()
except AlreadyLocked:
logger.debug('Lock already in place. Quitting')
return
except LockTimeout as e:
logger.debug('%s', e)
return
finally:
if lock.is_locked():
lock.release()
class NoseLock(Plugin):
name = 'lock'
def options(self, parser, env=os.environ):
super(NoseLock, self).options(parser, env=env)
# getting reasonable defaults
app_dir = os.getcwd()
app_name = os.path.basename(app_dir)
default_lock_file = os.path.join('/tmp', app_name)
parser.add_option(
'--lock-file', action='store',
default=default_lock_file,
dest='lock_file',
help='Use this file to acquire lock (default: {0})'.format(
default_lock_file))
def configure(self, options, conf):
super(NoseLock, self).configure(options, conf)
if not self.enabled:
self.lock = None
else:
lock_file = options.lock_file
self.lock = FileLock(lock_file)
if self.lock.is_locked():
owner = get_owner(lock_file + '.lock')
if owner:
print ('User {0} already running the tests, '
'please keep calm.').format(owner)
try:
self.lock.acquire()
log.info('File {0} locked.'.format(self.lock.lock_file))
print 'LOCK:', lock_file
except KeyboardInterrupt:
print '\nYou are so impatient today!\nBye.'
sys.exit(1)
def finalize(self, result):
if self.lock:
import pudb; pudb.set_trace() # DEBUG
print 'UNLOCK', self.lock.lock_file
log.info('Unlocking {0}.'.format(self.lock.lock_file))
self.lock.release()
def _acquire_lock(config):
'''
Determines if the executor is configured to run fetched jobs one after the other.
If this is the case, then check if another script instance is still running.
'''
if config.getboolean("Execution", "serialize"):
lock = FileLock(config.get("Execution", "pidfile"))
try:
if lock.is_locked() and not lock.i_am_locking():
logger.debug("Already locked")
return False
else:
lock.acquire()
logger.debug("Got the script lock")
except Exception as e:
logger.error("ERROR locking. " + str(e))
return False
return True
class DaemonLock(object):
"""Provides a lock for the MCVirt daemon"""
def __init__(self, timeout=2):
"""Create the lock file and lock file object and obtains a lock"""
# Create lock file, if it does not exist
if not os.path.isfile(DirectoryLocation.LOCK_FILE):
if not os.path.isdir(DirectoryLocation.LOCK_FILE_DIR):
os.mkdir(DirectoryLocation.LOCK_FILE_DIR)
open(DirectoryLocation.LOCK_FILE, 'a').close()
# Attempt to lock lockfile
self.lockfile_object = FileLock(DirectoryLocation.LOCK_FILE)
# Check if lockfile object is already locked
if self.lockfile_object.is_locked():
raise MCVirtLockException(
'An instance of MCVirt is already running')
else:
self.lockfile_object.acquire(timeout=timeout)
atexit.register(self.lockfile_object.release)
class Command(BaseCommand):
"""
"""
help = ""
pidfile = None
option_list = BaseCommand.option_list + (
make_option('--pidfile', action='store', dest='pidfile',
default=os.path.join(settings.PROJECT_ROOT, 'collector_query.pid'),
help='PID file for this command, to avoid multiple executions. '\
'Default: %s' % os.path.join(settings.PROJECT_ROOT,
'collector_query.pid')),
make_option('--host', action='store', dest='host',
default=settings.COLLECTOR_CONF['host'],
help='Host to run query server on. Default: %s' %
settings.COLLECTOR_CONF['host']),
make_option('--port', action='store', dest='port',
default=settings.COLLECTOR_CONF['port'],
type=int, help='Port to bind the query server on. Default: %d' %
settings.COLLECTOR_CONF['port']),
make_option('--time-lapse', action='store', dest='time-lapse',
default=settings.COLLECTOR_CONF['query_time-lapse'],
type=int, help='How many seconds between update lapses. '\
'Default: %d' %
settings.COLLECTOR_CONF['query_time-lapse']),
make_option('--rrd-dir', action='store', dest='rrd-dir',
default=settings.RRD_DIR, help='Directory to store RRD files. '\
'Default: %s' % settings.RRD_DIR),
make_option('-w', '--workers', action='store', dest='workers',
type=int, default=settings.COLLECTOR_CONF['query_workers'],
help="How many thread workers to use to perform request to RPC "\
"server. Default: %d" %
settings.COLLECTOR_CONF['query_workers']),
)
cache = CacheWrapper(cache)
def __lock_pid(self, path):
"""
"""
logger.debug("PID file at '%s'" % path)
self.pidfile = FileLock(path)
if self.pidfile.is_locked():
raise AlreadyRunningError("PID file is already locked (other "\
"process running?).")
try:
self.pidfile.acquire(
timeout=settings.COLLECTOR_CONF['pidlock_timeout'])
except LockTimeout:
logger.exception("Can't lock PID file:")
raise ContextException("Failed to lock pidfile.")
def __release_pid(self):
"""
"""
logger.debug("Releasing PID file.")
if self.pidfile:
self.pidfile.release()
def __create_queue(self):
"""
"""
queue = Queue.Queue()
active_servers = self.cache.get_list('server_active_ids',
'server_%d', Server, Server.objects.filter, active=True)
for s in active_servers:
tasks = self.cache.do().get('server_%d_tasks' % s.id, [])
if not tasks:
for r in s.reports.all():
sections = self.cache.get_list('server_%d_sections' % s.id,
'section_%d', Section, r.sections.all)
for se in sections:
variables = self.cache.get_list('server_%d_variables' %
s.id, 'variable_%d', Variable,
se.variables.filter, current=False)
[tasks.append((s, se, v)) for v in variables]
self.cache.do().set('server_%d_tasks' % s.id, tasks)
[queue.put(t) for t in tasks]
return queue
def handle(self, *args, **options):
"""
"""
logger.info("** Collector RPC client started. **")
workers = []
try:
self.__lock_pid(options['pidfile'])
queue = self.__create_queue()
for id in range(options['workers']):
workers.append(QueryWorker(id=id, queue=queue,
host=options['host'], port=options['port'],
time_lapse=options['time-lapse'],
rrd_dir=options['rrd-dir']))
[w.start() for w in workers]
[w.join() for w in workers]
exit(SUCCESS)
except AlreadyRunningError:
exit(ALREADY_RUNNING_ERROR)
except ContextException:
exit(CONTEXT_EXCEPTION)
finally:
self.__release_pid()
logger.info("** Collector RPC client finished. **")
args = parser.parse_args()
return args
def main():
args = argument_parser()
logging.basicConfig(
format="%(asctime)s %(name)s %(levelname)s: %(message)s", level=getattr(logging, args.loglevel.upper(), None)
)
# Use function accordingly to action specified
try:
output = args.script_action(args)
if output:
print(output)
except:
print("ERROR: failure running with script action")
print("ERROR:", sys.exc_info())
sys.exit(-1)
if __name__ == "__main__":
# Initialise locking
lockfile = FileLock("/var/lock/elasticsearch-backup.lock")
if lockfile.is_locked():
print("ERROR: /var/lock/elasticsearch-backup.lock is already locked," "probably we're already running")
sys.exit(1)
else:
with lockfile:
main()
def __call__(self):
self.call_count += 1
lock = FileLock(start_up.LOCK_FILE_NAME)
self.lock_was_held = lock.is_locked()
class Server(object):
def __init__(self, path):
self.defines = dict(DEFINES)
self.defines.update({
'path': path,
})
self.lock = FileLock(os.path.join(path, DEFINES['lock_filename']))
self.load_properties(path)
self.defines['world'] = self.properties.get('level-name', 'world')
self.rcon_password = self.properties.get('rcon.password')
if getpass.getuser() != self.defines['username']:
self.call_switch_user = True
else:
self.call_switch_user = False
def load_properties(self, path):
self.properties = properties = {}
properties_path = os.path.join(path, 'server.properties')
if not os.path.exists(properties_path):
return
try:
with open(properties_path) as f:
properties_file = f.read()
except:
raise # deal with that later
for line in properties_file.split('\n'):
if line.startswith('#') or not line.strip() or not '=' in line:
continue
key, value = line.split('=', 1)
properties[key] = value
def rcon(self, cmd, defines=None):
"""
Send a rcon message to the server and return its response
"""
shell_args = [
'/usr/bin/mcrcon', '-H', 'localhost', '-p',
self.rcon_password, cmd]
shell_cmd = ' '.join(shell_args)
try:
return subprocess.check_call(shell_args)
except subprocess.CalledProcessError as err:
return err.output or ''
def call(self, cmd, defines=None):
"""Call a shell command template string for the server at `path`"""
if defines:
defines = dict(defines)
defines.update(self.defines)
else:
defines = self.defines
cmd = cmd % defines
if self.call_switch_user:
wrapper = "su - %(username)s -c \"%%s\"" % defines
cmd = wrapper % cmd
svanillaubprocess.call(cmd, shell = True)
def lock_exists(self):
"""Returns whether there's a lock"""
return self.lock.is_locked()
def ensure_lock(self):
"""Exit immediately if the lock doesn't exist"""
if not self.lock_exists():
print "%(service)s is not running" % self.defines
sys.exit(0)
def wait_for(self, fn, seconds=10):
"""Wait for fn to be true. Returns False after `seconds`"""
end_time = datetime.now() + timedelta(seconds=seconds)
while datetime.now() < end_time:
if fn():
return True
return False
def start(self):
"""Start the service"""
if self.lock_exists():
print "%(service)s is already running" % self.defines
sys.exit(0)
defines = {'service_runner': __file__}
defines.update(self.defines)
print "Starting %(service)s" % defines
env = os.environ.copy()
env['MC_JAR_NAME'] = defines['service']
subprocess.Popen(
"python %(service_runner)s %(path)s run-server" % defines,
env=env,
shell=True)
time.sleep(0.5)
if self.wait_for(lambda: not self.lock_exists(), seconds=1):
print "Failed to start %(service)s" % self.defines
else:
print "Started %(service)s" % self.defines
def server_run(self):
"""
Wrap the server process
"""
cmd = ('cd %(path)s; '
'%(java)s -Xmx1024M -Xms1024M -jar %(service)s nogui >& /dev/null'
) % self.defines
with self.lock:
subprocess.check_call(cmd, shell=True)
def stop(self):
"""Nicely shuts the service down"""
self.ensure_lock()
print "Saving map..."
self.save(msg="Shutting down in 10 seconds. Saving map...")
self.say("Shutting down NOW")
print self.rcon("stop")
if self.wait_for(lambda: not self.lock_exists()):
msg = "Service %(service)s shut down"
else:
msg = "Service %(service)s shut down"
print msg % self.defines
def say(self, msg):
"""Broadcast a message on the server"""
self.rcon('say "%s"' % msg)
def save(self, msg=None):
"""Save the server, optionally with an informative message"""
if msg:
self.say(msg)
self.rcon("save-all")
def backup(self):
response = self.save("Backing up the server...")
if not response:
print "Error backing up (no response from rcon)"
return
print ">>>", response
try:
self.rcon("save-off")
self.call(os.path.join(
"tar -chzvf %(backup_dir)s", "%(world)s-%(day)s.tgz %(path)s", "%(world)s"), {
'day': datetime.now().strftime('%y-%m-%d'),
})
finally:
self.rcon("save-on")
def status(self):
"""Print the service's status"""
if self.lock_exists():
msg = "%(service)s is running"
else:
msg = "%(service)s is stopped"
print msg % self.defines
def reload_(self):
"""Reload the service's configuration and plugins"""
self.ensure_lock()
self.rcon("reload")
def restart(self):
"""Restart the service"""
if self.lock_exists():
self.stop()
self.start()
def shell(self):
"""
Open an rcon shell
"""
self.ensure_lock()
proc = subprocess.Popen(
['mcrcon', '-H', 'localhost', '-p', self.rcon_password, '-t'],
stdout=sys.stdout, stdin=sys.stdin)
proc.wait()
from .config import DB_CONN_STRING
init_db(DB_CONN_STRING)
if arguments['generate'] or arguments['cron'] or arguments['serve']:
from . import generate
generate.csv_override = open(arguments['--override']) if arguments['--override'] else None
if arguments['generate']:
from .generate import QualtricsStats
QS = QualtricsStats(arguments['<survey_xml_spec>'])
print(QS.run())
if arguments['cron']:
from .cron import cron
lock = FileLock("/tmp/qualtrics_stats.lock")
if lock.is_locked():
logging.error('Lockfile locked, exiting.')
sys.exit(1)
with lock:
cron()
if arguments['serve']:
from .server import serve
from .config import SERVER_LISTEN_ADDR
serve(SERVER_LISTEN_ADDR)
if arguments['gen_API_key']:
from .db import gen_API_key
print(gen_API_key())
def run(self):
self.busy = True
print "--ID " + self.processing + " Running FilterWrite"
if self.mode == "encode":
length = self._filter_input[0]
dict_converted = self._filter_input[1]
converted_data = self._filter_input[2]
rank_alphabet = self._filter_input[3]
filename = self._filter_input[4]
output_file = filename + ".d2f"
lock_file = None
if os.path.isfile(output_file):
lock_file = FileLock(output_file)
status = lock_file.is_locked()
while status:
status = lock_file.is_locked()
lock_file.acquire()
file_open = open(output_file, "wb")
file_open.write(converted_data)
file_open.close()
if lock_file:
lock_file.release()
result = str(length) + "_"
for i in rank_alphabet:
temp = dict_converted[i]
length = len(dict_converted[i])
count_zero = length % 8
if count_zero != 0:
temp = '0' * count_zero + temp
temp = int(temp, 2)
result += str(length) + "-" + str(temp) + "" + "/|" + i
if rank_alphabet.index(i) < len(rank_alphabet) - 1:
result += "=*"
lock_file = None
output_file = filename + ".d2c"
if os.path.isfile(output_file):
lock_file = FileLock(output_file)
status = lock_file.is_locked()
while status:
status = lock_file.is_locked()
lock_file = FileLock(output_file)
lock_file.acquire()
file_open = open(output_file, "wb")
file_open.write(result)
file_open.close()
if lock_file:
lock_file.release()
if self.mode == "decode":
filename = self._filter_input[0]
converted_data = self._filter_input[1]
temp_pos = filename.rfind('.')
filename = filename[:temp_pos]
# temp = False
lock_file = None
# print filename
if os.path.isfile(filename):
lock_file = FileLock(filename)
status = lock_file.is_locked()
while status:
status = lock_file.is_locked()
lock_file.acquire()
# temp = True
file_open = open(filename, "wb")
file_open.write(converted_data)
file_open.close()
if lock_file:
lock_file.release()
self.taken = True
print "--ID " + self.processing + " Finish"
self.busy = False
for submission in submissions:
try:
submission.started = datetime.now()
submission.status = 'running'
submission.save()
print submission.started, "Starting", submission.token, "from", submission.ip_address
check_call(analysis_script + ' ' + ' -i ' +
submission.input_vcf_path + ' -v ' +
str(submission.min_variant_quality) + ' -d ' +
str(submission.min_quality_depth) + ' -w ' +
str(submission.homozyg_window_size) + ' -c ' +
str(submission.heterozyg_calls) + ' -z ' + ' &> ' +
submission.full_upload_path + '/analysis.log',
shell=True)
except CalledProcessError, e:
submission.status = 'failed'
else:
submission.status = 'completed'
finally:
submission.finished = datetime.now()
print submission.finished, "Finished", submission.token
submission.save()
if __name__ == "__main__":
if lock.is_locked():
exit(0)
else:
with lock:
manager.run()
def __call__(self):
self.call_count += 1
lock = FileLock(start_up.LOCK_FILE_NAME)
self.lock_was_held = lock.is_locked()
sh = logging.StreamHandler(sys.stdout)
sh.setFormatter(formatter)
logger.addHandler(sh)
else:
trfh = logging.handlers.TimedRotatingFileHandler(
filename = config.get('log', 'log_file'),
when = 'midnight',
backupCount = 5,
utc=True
)
trfh.setFormatter(formatter)
logger.addHandler(trfh)
# Try to acquire the lock
lock = FileLock(config.get('lock', 'lock_file'))
if (lock.is_locked()):
logger.debug('Could not acquire lock')
sys.exit()
else:
lock.acquire()
logger.info('Twitter Stream starting up..')
l = Listener(
tweet_filter = args.filter,
redis_host = args.redis_host,
redis_port = int(args.redis_port)
)
auth = OAuthHandler(args.consumer_key, args.consumer_secret)
auth.set_access_token(args.access_token, args.access_token_secret)
#!/usr/bin env python
from lockfile import FileLock
import sys
lock = FileLock(__file__)
if lock.is_locked()==True: sys.exit()
lock.acquire()
for i in range(10000):
print i**2
