def move_to_bad_logs(filename):
"""moves filename into BAD_LOGS dir"""
bad_logs_dir = os.path.join(os.path.dirname(filename), '..', 'bad_logs')
bad_logs_dir = os.path.abspath(bad_logs_dir)
mkdir_p(bad_logs_dir)
name = os.path.split(filename)[1]
dst_file = os.path.join(bad_logs_dir, name)
log.debug("%s => %s", filename, dst_file)
os.rename(filename, dst_file)
def move_to_bad_logs(filename):
"""moves filename into BAD_LOGS dir"""
bad_logs_dir = os.path.join(os.path.dirname(filename), '..', 'bad_logs')
bad_logs_dir = os.path.abspath(bad_logs_dir)
mkdir_p(bad_logs_dir)
name = os.path.split(filename)[1]
dst_file = os.path.join(bad_logs_dir, name)
log.debug("%s => %s", filename, dst_file)
os.rename(filename, dst_file)
def write_to_json(events_dir, data):
"""writes data to a json file; the file name is:
<EVENTS_DIR>/event/instance,
event and instance are provided by data itself"""
event = data['eventName']
instance = data['instances']
filename = os.path.join(events_dir, event, instance)
mkdir_p(os.path.dirname(filename))
if not os.path.exists(filename):
with open(filename, 'w') as f_out:
json.dump(data, f_out)
elif data['eventTime'] > get_time_from_file(filename):
# replace old event with current one
with open(filename, 'w') as f_out:
json.dump(data, f_out)
def write_to_json(events_dir, data):
"""writes data to a json file; the file name is:
<EVENTS_DIR>/event/instance,
event and instance are provided by data itself"""
event = data['eventName']
instance = data['instances']
filename = os.path.join(events_dir, event, instance)
mkdir_p(os.path.dirname(filename))
if not os.path.exists(filename):
with open(filename, 'w') as f_out:
json.dump(data, f_out)
elif data['eventTime'] > get_time_from_file(filename):
# replace old event with current one
with open(filename, 'w') as f_out:
json.dump(data, f_out)
def write_to_disk(cache_dir, key):
"""write key to disk in cache_dir"""
dst = os.path.join(cache_dir, key.name)
mkdir_p(os.path.dirname(dst))
# key.get_contents_to_filename() is a blocking function,
# if we try to download non existing files, it will hang here
# it works only on unix systems
signal.signal(signal.SIGALRM, _timeout)
if not os.path.exists(dst):
log.debug('downloading: {0}'.format(key.name))
signal.alarm(GET_CONTENTS_TO_FILENAME_TIMEOUT)
try:
key.get_contents_to_filename(dst)
except TimeoutException:
log.debug('timeout downloading: {0}'.format(key.name))
else:
# file is already cached locally
log.debug('{0} is already cached'.format(key.name))
def write_to_disk(cache_dir, key):
"""write key to disk in cache_dir"""
dst = os.path.join(cache_dir, key.name)
mkdir_p(os.path.dirname(dst))
# key.get_contents_to_filename() is a blocking function,
# if we try to download non existing files, it will hang here
# it works only on unix systems
signal.signal(signal.SIGALRM, _timeout)
if not os.path.exists(dst):
log.debug('downloading: {0}'.format(key.name))
signal.alarm(GET_CONTENTS_TO_FILENAME_TIMEOUT)
try:
key.get_contents_to_filename(dst)
except TimeoutException:
log.debug('timeout downloading: {0}'.format(key.name))
else:
# file is already cached locally
log.debug('{0} is already cached'.format(key.name))
