def es_store_record(record):
try:
logger.debug(f"es_store_record in index: {es_index_name} record {record}")
es.index(index=es_index_name, body=record)
except Exception as ex:
logger.error(ex, exc_info=True)
def piped_execute(cmd1, cmd2):
"""Pipe output of cmd1 into cmd2."""
logger.debug("Piping cmd1='%s' into...", ' '.join(cmd1))
logger.debug("cmd2='%s'", ' '.join(cmd2))
try:
p1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError as e:
logger.error("Pipe1 failed - %s ", e)
raise
# NOTE(dosaboy): ensure that the pipe is blocking. This is to work
# around the case where evenlet.green.subprocess is used which seems to
# use a non-blocking pipe.
flags = fcntl.fcntl(p1.stdout, fcntl.F_GETFL) & (~os.O_NONBLOCK)
fcntl.fcntl(p1.stdout, fcntl.F_SETFL, flags)
try:
p2 = subprocess.Popen(cmd2, stdin=p1.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError as e:
logger.error("Pipe2 failed - %s ", e)
raise
p1.stdout.close()
stdout, stderr = p2.communicate()
return p2.returncode, stderr
def execute(cmd):
cmd = [str(c) for c in cmd]
logger.debug("Execute cmd='%s'", ' '.join(cmd))
try:
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
return p.communicate()
except Exception as ex:
logger.error("Execute cmd error %s", ex)
def list_beatport_directory(self, directory):
path = "/MP3/BEATPORT__AND__WEBSITE_SECTION/"
self.ftp.cwd(path)
self.ftp.cwd(directory)
for entry in (path for path in self.ftp.nlst()
if path not in ('.', '..')):
try:
logger.debug("Entering directory: {}".format(entry))
self.ftp.cwd(entry)
ftpFile = FtpFile("BEATPORT__AND__WEBSITE_SECTION", entry,
path + directory)
largest = 0
for filename in (path for path in self.ftp.nlst()
if path not in ('.', '..')):
logger.debug("entry {}".format(entry))
logger.debug("sub_entry {}".format(filename))
logger.debug("path {}".format(path))
song_logger.info(
directory + "," +
filename.replace('-www.groovytunes.org', ''))
logger.debug("getting timestamp of {}".format(filename))
size = self.ftp.size(filename)
if (largest < size):
largest = size
ftpFile.size = size
logger.debug(ftpFile)
es_store_record(ftpFile.toDict())
result = [
fav_element for fav_element in banned
if fav_element.upper() in entry.upper().replace(' ', '_')
]
if (len(result) == 0):
result = [
fav_element for fav_element in favourites
if fav_element.upper() in entry.upper().replace(
' ', '_')
]
if (len(result) > 0):
if (len(
difflib.get_close_matches(
entry, self.matcher_list)) == 0):
logger.info(f"Adding to BT q {entry}")
self.queue_bt.append(ftpFile)
self.matcher_list.append(entry)
self.ftp.cwd('..')
except Exception:
logger.error("Listing error: ", exc_info=True)
def send_plaintext_mail(message):
try:
with smtplib.SMTP_SSL(smtp_server, port, context=context) as server:
newline = "\n"
message = f"Subject: -= MUSIC DL =- {newline}To: {sender_email}{newline}From: {sender_email}{newline}{message}"
server.login(sender_email, password)
server.sendmail(sender_email, sender_email, message)
except Exception as ex:
logger.error(ex, exc_info=True)
def get_audio_info(path):
try:
audio = MP3(path)
length_minutes = time.gmtime(audio.info.length).tm_min
bitrate = audio.info.bitrate / 1000
return length_minutes, bitrate
except Exception as ex:
logger.error(ex, exc_info=True)
send_plaintext_mail(f"error get_audio_info of {path}")
raise ex
def _connect(self):
logger.debug("connect")
try:
self.conn = mariadb.connect(user=self.user,
password=self.pwd,
host=self.host,
port=3307,
database="rigodetools")
except Exception as ex:
logger.error(ex, exc_info=True)
print(ex)
def list_0day_directory(self, directory):
path = "/MP3/0-DAY/"
self.ftp.cwd(path)
self.ftp.cwd(directory)
for entry in (path for path in self.ftp.nlst()
if path not in ('.', '..')):
try:
logger.debug("Entering directory: " + entry)
self.ftp.cwd(entry)
ftpFile = FtpFile("0-DAY", entry, path + directory)
largest = 0
for filename in (path for path in self.ftp.nlst()[1:]
if path not in ('.', '..')
): #first entry is always a sub directory
logger.debug("entry " + entry)
logger.debug("sub_entry " + filename)
logger.debug("path " + path)
song_logger.info(
directory + "," +
filename.replace('-www.groovytunes.org', ''))
size = self.ftp.size(filename)
if (largest < size):
largest = size
ftpFile.size = size
es_store_record(ftpFile.toDict())
result = [
fav_element for fav_element in banned
if fav_element.upper() in entry.upper().replace(' ', '_')
]
if (len(result) == 0):
result = [
fav_element for fav_element in favourites
if fav_element.upper() in entry.upper().replace(
' ', '_')
]
if (len(result) > 0):
if (len(
difflib.get_close_matches(
entry, self.matcher_list)) == 0):
logger.info(f"Adding to 0-day q {entry}")
self.queue_oday.append(ftpFile)
self.matcher_list.append(entry)
self.ftp.cwd('..')
except Exception:
logger.error("Listing error: ", exc_info=True)
def generate_m3u(directory):
try:
fullPath = os.path.join(directory, "!generated.m3u")
if (os.path.exists(fullPath)):
os.remove(fullPath)
m3u = open(fullPath, "x")
try:
for file in files(directory):
m3u.write(str(file) + "\n")
finally:
m3u.close()
except Exception as ex:
logger.error(ex, exc_info=True)
def run():
logger.info("start monitor vm memory")
while True:
if system_memory_pressure_critical() or is_midnight():
try:
with LibvirtWrapper() as client:
instances = client.list()
for instance in instances:
if instance.need_drop_cache():
with VirshBalloon(instance) as balloon:
balloon.deflate()
except Exception as ex:
logger.error(ex)
logger.error(traceback.format_stack())
logger.info('monitor sleep %d seconds.' % INTERVAL)
time.sleep(INTERVAL)
def send_html_mail(message):
try:
message = MIMEMultipart('alternative')
message["Subject"] = "-= MUSIC DL =-"
message["From"] = sender_email
message["To"] = [sender_email]
part_html = MIMEText(message, "html")
message.attach(part_html)
with smtplib.SMTP_SSL("smtp.gmail.com", port,
context=context) as server:
server.login(sender_email, password)
raw = base64.urlsafe_b64encode(
message.as_string().encode('UTF-8')).decode('ascii')
server.sendmail(sender_email, sender_email, raw)
except Exception as ex:
logger.error(ex, exc_info=True)
raise Exception(ex)
def clean_download_directory(directory):
logger.debug(f"Cleaning directory: {directory}")
for filename in os.listdir(directory):
fullpath = os.path.join(directory,filename)
if (os.path.isfile(fullpath)):
if (filename.endswith('.mp3')):
try:
length_minutes, bitrate = get_audio_info(fullpath)
if (length_minutes > 25) or (length_minutes < 4) or (bitrate < 192):
os.remove(fullpath)
except Exception as ex:
logger.error(f"Audio info couldn't be determined. Keeping the file {fullpath}")
else:
clean_download_directory(fullpath)
if not [f for f in os.listdir(directory) if f.endswith('.mp3')]:
shutil.rmtree(directory)
else:
generate_m3u(directory)
def es_create_index():
settings = {
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"members": {
"dynamic": "strict",
"properties": {
"group": {
"type": "text"
},
"date": {
"type": "date",
"format": "yyyy.MM.dd HH:mm:ss"
},
"directory": {
"type": "text"
},
"filename": {
"type": "text"
},
"full_filename": {
"type": "text"
},
"pretty_filename": {
"type": "text"
}
}
}
}
}
try:
if not es.indices.exists(es_index_name):
es.indices.create(index=es_index_name, ignore=400, body=settings)
logger.debug('Index created')
except Exception as ex:
logger.error(ex, exc_info=True)
from codetiming import Timer
import sys
try:
today = datetime.now() - timedelta(1)
today_directory = today.strftime("%m%d")
if (len(sys.argv) == 2):
today_directory = sys.argv[1]
logger.debug(today_directory)
print("Crawling {}".format(today_directory))
timer = Timer("ftp", text="Finished in {minutes:.1f} minutes")
with FtpCrawler() as ftpcrawler:
timer.start()
ftpcrawler.list_beatport_directory(today_directory)
ftpcrawler.list_0day_directory(today_directory)
ftpcrawler.download_queue_bt(today_directory)
ftpcrawler.download_queue_oday(today_directory)
timer.stop()
download_time = Timer.timers["ftp"]
except Exception as ex:
logger.error(ex, exc_info=True)
print(ex)
raise Exception("Report abnormal termination to the NAS scheduler")