def store(hpath, segment, tmpdir):
'''
Store retrieved record to HDFS.
:param hpath : Destination folder in HDFS.
:param segment: Incoming segment from Kafka cluster.
:param tmpdir : Path to temporary directory on local file system.
'''
logger = logging.getLogger('SHIELD.WORKER.{0}.STORE'.format(
current_process().name))
try:
timestamp = segment[0].strftime('%Y%m%d%H%M')
logger.info('Store segment "{0}" to HDFS...'.format(timestamp))
localfile = os.path.join(tmpdir, timestamp)
with open(localfile, 'w') as fp:
[fp.write(x + '\n') for x in segment[1]]
logger.info('Store segment locally: {0}'.format(localfile))
# .............................put local file to HDFS
popen('hdfs dfs -put {0} {1}'.format(localfile, hpath), raises=True)
logger.info('File stored at "{0}/{1}".'.format(hpath, timestamp))
os.remove(localfile)
except Exception as exc:
logger.error('[{0}] {1}'.format(exc.__class__.__name__, exc.message))
def run_manage_action(self, netapp_identifier: str, action_name: str):
command = self.db.get_netapp_action(netapp_identifier, action_name)
if command:
cd(self.home + '/netapps/' + netapp_identifier)
popen(command)
return None
else:
return "Error: invalid parameters"
def beat_mode(argument):
celery_bin = runner_config.CELERY_BIN
cmd = [celery_bin, 'beat', '-A celery_app']
if argument.celery_config:
cmd.append('--config=%s' % argument.celery_config)
if runner_config.LOGGER_LEVEL:
cmd.append('--loglevel=%s' % runner_config.LOGGER_LEVEL)
if runner_config.LOGGER_FILE:
cmd.append('--logfile=%s' % runner_config.LOGGER_FILE)
if argument.schedule:
cmd.append('--schedule=%s' % argument.schedule)
c = ' '.join(cmd)
utils.popen(c)
return 0
def worker_mode(argument):
celery_bin = runner_config.CELERY_BIN
c = [celery_bin, 'worker', '-A celery_app']
if argument.celery_config:
c.append('--config=%s' % argument.celery_config)
if runner_config.LOGGER_LEVEL:
c.append('--loglevel=%s' % runner_config.LOGGER_LEVEL)
else:
c.append('--loglevel=INFO')
if runner_config.LOGGER_FILE:
c.append('--logfile=%s' % runner_config.LOGGER_FILE)
if argument.beat:
c.append('--beat')
c = ' '.join(c)
utils.popen(c)
return 0
def log_mode(argument):
logf = 'worker.log'
if 'LOGGER_FILE' in dir(runner_config):
logf = runner_config.LOGGER_FILE
follow = False
if argument.follow:
follow = True
if os.path.exists(logf):
if follow:
c = 'tail -f %s' % logf
utils.popen(c)
else:
fh = open(logf)
for l in fh:
print(l)
fh.close()
else:
print('Log file %s does not exists!' % logf)
return 0
def parse_dna(self):
cmd = "dna_status -i {dna} 2>&1".format(dna=self.dna)
ret, out, err = popen(cmd)
self.logger.debug("parse_dna: cmd: %s out: %s, ret: %s, "
"err: %s" % (cmd, out, ret, err))
try:
if ret == 0:
out = out.strip("\n").split("\n")[-1]
self.dna_duration = int(out.split(" ")[1])
self.dna_type = out.split(" ")[0].lower()
except:
self.looger.error("parse_dna failed", exc_info=True)
ret = 1
return ret
def parse_dna(self):
cmd = "dna_status -i {dna} 2>&1".format(dna=self.dna)
ret, out, err = popen(cmd)
self.logger.debug("parse_dna: cmd: %s out: %s, ret: %s, "
"err: %s" % (cmd, out, ret, err))
try:
if ret == 0:
out = out.strip("\n").split("\n")[-1]
self.dna_duration = int(out.split(" ")[1])
self.dna_type = out.split(" ")[0].lower()
except:
self.looger.error("parse_dna failed", exc_info=True)
ret = 1
return ret
def download(self):
success = False
container, remote_path = self.parse_dna_url(self.task.dna_url)
cmd = "swift download -A {auth} -U {user} -K {key} {container} -o {dna_file} "\
"{remote_path}".format(auth=self.config['swift_auth'],
user=self.config['swift_user'],
key=self.config['swift_key'],
container=container,
dna_file=self.dna,
remote_path=remote_path
)
ret, _, err = popen(cmd)
if ret or err:
self.logger.error("download falied cmd: %s, url: %s, ret: %s, err: %s" %
(cmd, self.task.dna_url, ret, err))
else:
self.logger.info("download success, url:%s " % self.task.dna_url)
success = True
return success
def download(self):
success = False
container, remote_path = self.parse_dna_url(self.task.dna_url)
cmd = "swift download -A {auth} -U {user} -K {key} {container} -o {dna_file} "\
"{remote_path}".format(auth=self.config['swift_auth'],
user=self.config['swift_user'],
key=self.config['swift_key'],
container=container,
dna_file=self.dna,
remote_path=remote_path
)
ret, _, err = popen(cmd)
if ret or err:
self.logger.error(
"download falied cmd: %s, url: %s, ret: %s, err: %s" %
(cmd, self.task.dna_url, ret, err))
else:
self.logger.info("download success, url:%s " % self.task.dna_url)
success = True
return success
def get_usage(cmd):
return utils.popen('git {} -h'.format(cmd))
def popen(self, command, fatal = True): command = self.get_remote_command(command) return utils.popen(command, fatal, self.config.verbose, self.logfile)
def slice_popen(self, command, user = '******', key = None, fatal = True):
command = self.get_remote_command(command, user, key)
return utils.popen(command, fatal, logfile = self.logfile)
def host_popen(self, command, fatal = True, logfile = None):
if logfile is None:
logfile = self.logfile
command = self.get_host_command(command)
return utils.popen(command, fatal, self.config.verbose, logfile)