def __init__(self, conf):
self.config = conf
#set logging configuration
logging.basicConfig(format='%(levelname)s:%(asctime)s - %(message)s')
self.log = logging.getLogger()
self.log.setLevel(logging.getLevelName(self.config.log_level))
self.dbx = DropboxHandler(self.config.access_token, self.config.dropbox_timeout, self.config.dropbox_chunck)
def __init__(self, bot, dispatcher, database_handler, token): self.dispatcher = dispatcher # queue for async jobs self.job_queue = JobQueue(bot) # Where to get pictures from: local filesystem(local) or Dropbox storage (DB) self.pic_source = sr["pic_source"] self.database_handler = database_handler super(UserCommandHandler, self).__init__(token, self.database_handler) self._addHandlers() if self.pic_source == "DB": self.DB_file_updater_thread = None # a thread that updates files self.dropbox_handler = DropboxHandler(self.database_handler) self._updateDBFiles() elif self.pic_source == "local": self.local_cleaner_job = None self._startLocalCleanerJob() self._initializeSubscriptionJobs()
def pushEquippedFileSystem(self, cloudService):
# initialize chosen session
if cloudService == 'drive':
cloudHandler = GoogleDriveHandler()
elif cloudService == 'dropbox':
# get access token if exists
accessToken = self.getSetting('dbAccessToken')
if accessToken is None:
print("> Initializing dropbox for first time")
cloudHandler = DropboxHandler(accessToken)
# update saved access code
self.setSetting('dbAccessToken', cloudHandler.access_token)
else:
print("> Error: Unrecognized cloud service")
return
# get UUIDs for equipped system
equippedSystem = self.getEquippedSystem()
if equippedSystem is None:
return
info = self.getSystemInfo(equippedSystem)
if info is None:
print("> Error: No info found for system '{}'".format(
equippedSystem))
filesInfo = info['files'] if 'files' in info else []
fileUUIDs = {}
for fileName, fInfo in filesInfo.items():
if 'uuid' in fInfo:
fileUUIDs[fileName] = fInfo['uuid']
if len(fileUUIDs) == 0:
print(
"> No encrypted files to push! Encrypt the equipped filesystem using the 'encrypt' command."
)
return
# push and timestamp (seconds) each file being pushed
for fileName, uuid in fileUUIDs.items():
# push file
if cloudService == 'drive':
res = cloudHandler.upsert_file(uuid, "crypt/")
else:
res = cloudHandler.upsert_file(uuid, "crypt/{}".format(uuid),
uuid)
if res is not None:
print("> Upload successful.")
else:
print("> Upload failed.")
return
filesInfo[fileName]['pushed'] = round(time.time())
# update the database
systems = self.db.table('systems')
System = Query()
systems.update({'files': filesInfo}, System.name == equippedSystem)
class Processor():
def __init__(self, conf):
self.config = conf
#set logging configuration
logging.basicConfig(format='%(levelname)s:%(asctime)s - %(message)s')
self.log = logging.getLogger()
self.log.setLevel(logging.getLevelName(self.config.log_level))
self.dbx = DropboxHandler(self.config.access_token, self.config.dropbox_timeout, self.config.dropbox_chunck)
#upload file to dropbox
#if there is an error when uploading, files would be located in results path
def upload_file(self, df, idx):
filename = self.config.result_prefix + str(idx) + self.config.result_extension
file_from = self.config.result_folder + filename
file_to = self.config.dropbox_folder_upload + filename
print(file_to)
df.to_csv(file_from, index=False)
print(file_from)
self.log.info('Uploading file: %s', file_from)
upload = True
try:
self.dbx.upload_file(file_from, file_to)
except Exception as err:
self.log.error('Failed to upload %s\n%s', file_from, err)
upload = False
if upload:
os.remove(file_from)
return idx+1
#divide file into chunks and upload to dropbox
def save_data(self, df, idx):
chunks = get_chunks(self.config.output_size_mb, df)
if (chunks==0):
idx = self.upload_file(df, idx)
else:
for chunk in np.array_split(df, chunks):
idx = self.upload_file(chunk, idx)
return idx
#read file and return dataframe
def create_dataframe(self, local_path):
try:
df = pd.read_csv(local_path, header=0,
sep = get_delimiter(local_path, self.config.encoding_input),
usecols=['asin' , 'manufacturer','invalid'],
dtype={'asin':object,'manufacturer':object,'invalid':object},
encoding=self.config.encoding_input)
except Exception as err:
self.log.warning('Failed to process file:%s\n%s', local_path, err)
df = pd.read_excel(local_path,
header=0,
usecols=['asin' , 'manufacturer','invalid'],
dtype={'asin':object,'manufacturer':object,'invalid':object},
encoding=self.config.encoding_input)
return df
#list all files to process and for each append to df until reach threshold
#once the threshold is reach, the file is uploaded and start to create another df
def process_data(self):
files = self.dbx.list_recursive(self.config.dropbox_folder_download)
df = pd.DataFrame(data={})
idx=0
for file in files:
matcher = re.compile(self.config.file_regex)
file_dir = file[0]
filename = file[1]
file_path = file_dir + '/' + filename
if matcher.match(file[1]):
try:
local_path = self.config.data_folder + filename
self.dbx.download_file(file_path, local_path)
df2 = self.create_dataframe(local_path)
if (check_chunks(self.config.output_size_mb, df,df2)):
idx = self.save_data(df, idx)
df = df2
else:
df = df.append(df2)
os.remove(local_path)
except Exception as err:
self.log.error('Failed processing file %s\n%s', filename, err)
if (df.shape[0]>0):
self.log.info('Saving last chunck')
idx = self.save_data(df, idx)
def main(argv):
config_path = ''
import_data = False
process_data = False
category = ''
try:
opts, args = getopt.getopt(argv, 'hi:c:p:',
['config=', 'import=', 'process='])
except getopt.GetoptError:
info()
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
info()
sys.exit()
elif opt in ('-c', '--config'):
config_path = arg
elif opt in ('-i', '--import'):
import_data = bool(arg)
elif opt in ('-p', '--process'):
process_data = bool(arg)
try:
config = Config(config_path)
#set logging
logging.basicConfig(
format=
'%(levelname)s:%(asctime)s - %(pathname)s:%(lineno)d: %(message)s')
log = logging.getLogger()
log.setLevel(logging.getLevelName(config.log_level))
except Exception as ex:
print('There has been an error while initializing configuration.\n%s' %
(ex))
sys.exit(1)
try:
dbx = DropboxHandler(config.access_token, config.dropbox_timeout,
config.dropbox_chunck)
dbx.set_log(log)
except Exception as ex:
log.error(
'There has been an error while initializing dropbox handler.\n%s' %
(ex))
sys.exit(1)
try:
#import data from Google trends
if (import_data):
#download tickers file
if (not path.exists(config.tickers_folder)):
dbx.download_file(config.tickers_path, config.tickers_folder)
#download gtrends data
gt = GTrends(config.encoding, config.tz,
config.gtrends_timeout_connect,
config.gtrends_timeout_read, config.retries,
config.backoff_factor, config.geo, dbx)
gt.set_log(log)
download_all = gt.import_data(
config.tickers_folder, config.year_from, config.year_until,
config.categories, config.data_folder_monthly,
config.data_folder_daily, config.data_folder_monthly_dropbox,
config.data_folder_daily_dropbox)
print('download_all=%s' % str(download_all))
except Exception as ex:
log.error('There has been an error while importing data.\n%s' % (ex))
sys.exit(1)
try:
#process data and upload to dropbox
if (process_data):
p = Processor(config.prefix, config.output_size_mb, dbx)
p.set_log(log)
for category in config.categories:
category_type = category.split(':')
category_name = category_type[0]
category_type = category_type[1]
if (category_type == 'monthly'):
p.TL_data(config.data_folder_monthly_dropbox,
config.dropbox_folder_upload_monthly,
config.tmp_folder_monthly,
config.result_folder_monthly, 'monthly.csv',
category_name)
else:
p.TL_data(config.data_folder_daily_dropbox,
config.dropbox_folder_upload_daily,
config.tmp_folder_daily,
config.result_folder_daily, 'daily.csv',
category_name)
except Exception as ex:
log.error('There has been an error while processing data.\n%s' % (ex))
sys.exit(1)
sys.exit(0)
class UserCommandHandler(PicBotRoutines):
"""docstring for UserCommandHandler"""
def _command_method(func):
"""Decorator for functions that are invoked on commands. Ensures that the user is initialized."""
# @functools.wraps(func)
def wrapper(self, bot, update, *args, **kwargs):
# print("command method", func.__name__, ) # debug
# print("self",self)# debug
# print("command method", self, bot, update, args, kwargs, sep="||") # debug
chat_id = update.message.chat_id
log.info("Command method called!", func.__name__, "Chat_id: ", chat_id)
# Initialize user, if not present in DB
self.database_handler.initializeUser(chat_id=chat_id)
log.debug("User initialized")
lS = LanguageSupport(self.database_handler.getLang(chat_id)).languageSupport
# noinspection PyCallingNonCallable
func(self, bot, update, lS)
log.debug("Function completed")
return wrapper
def __init__(self, bot, dispatcher, database_handler, token):
self.dispatcher = dispatcher
# queue for async jobs
self.job_queue = JobQueue(bot)
# Where to get pictures from: local filesystem(local) or Dropbox storage (DB)
self.pic_source = sr["pic_source"]
self.database_handler = database_handler
super(UserCommandHandler, self).__init__(token, self.database_handler)
self._addHandlers()
if self.pic_source == "DB":
self.DB_file_updater_thread = None # a thread that updates files
self.dropbox_handler = DropboxHandler(self.database_handler)
self._updateDBFiles()
elif self.pic_source == "local":
self.local_cleaner_job = None
self._startLocalCleanerJob()
self._initializeSubscriptionJobs()
def _initializeSubscriptionJobs(self):
for chat_id in self.database_handler.getAllSubscribedUserIDs():
log.debug("_initializeSubscriptionJobs chat_id", chat_id)
self.createPeriodicSenderTask(chat_id)
def _updateDBFiles(self, bot=None, job=None):
if not self.DB_file_updater_thread or not self.DB_file_updater_thread.is_alive():
self.DB_file_updater_thread = self.dropbox_handler.updateFiles()
job = Job(self._updateDBFiles, interval=sr['file_update_period'], repeat=False)
else:
log.warning("The Dropbox updater thread hasn't finished yet. Consider increasing FILE_UPDATE_PERIOD in settings!")
job = Job(self._updateDBFiles, interval=10, repeat=False)
# create periodic job
self.job_queue.put(job)
def _startLocalCleanerJob(self):
"""
Creates a delayed async job that cleans database every now and then if local files get deeleted
:return:
"""
log.debug("_startLocalCleanerJob")
self.local_cleaner_job = job = Job(self._localCleanerThread, interval=LOCAL_CLEANER_PERIOD, repeat=True)
self.job_queue.put(job)
def _localCleanerThread(self, bot, job):
log.debug("_localCleanerThread")
local_files = self.getLocalFiles()
bd_files = set(self.database_handler.getFileList())
to_delete = bd_files.difference(local_files)
log.debug("to_delete", to_delete)
if to_delete:
self.database_handler.batchDeleteFiles(to_delete)
def _addHandlers(self):
self.dispatcher.add_handler(CommandHandler('start', self.command_start))
self.dispatcher.add_handler(CommandHandler('help', self.command_help))
self.dispatcher.add_handler(CommandHandler('about', self.command_about))
self.dispatcher.add_handler(CommandHandler('otherbots', self.command_otherbots))
self.dispatcher.add_handler(CommandHandler('gimmepic', self.command_gimmepic))
self.dispatcher.add_handler(CommandHandler('subscribe', self.command_subscribe))
self.dispatcher.add_handler(CommandHandler('unsubscribe', self.command_unsubscribe))
self.dispatcher.add_handler(CommandHandler('spamuncached', self.command_spamuncached))
# non-command message
self.dispatcher.add_handler(MessageHandler([Filters.text], self.messageMethod))
# unknown commands
self.dispatcher.add_handler(MessageHandler([Filters.command], self.unknown_command))
self.dispatcher.add_error_handler(self.error_handler)
log.info("Commands set!")
def setPeriod(self, bot, update, lS=None):
message = update.message.text
chat_id = update.message.chat_id
try:
new_period = int(message)
if not self.database_handler.getSubscribed(chat_id):
self.sendMessageCommandMethod(bot, update, "You're not subscribed yet! /subscribe first!")
else:
# If a period is too small
if new_period < MIN_PICTURE_SEND_PERIOD:
self.database_handler.setPeriod(chat_id, MIN_PICTURE_SEND_PERIOD)
self.sendMessageCommandMethod(bot, update,
"The minimum possible period is {0}.\nSetting period to {0}.".format(
str(MIN_PICTURE_SEND_PERIOD)))
# If a period is too big
elif new_period > MAX_PICTURE_SEND_PERIOD:
self.database_handler.setPeriod(chat_id, MAX_PICTURE_SEND_PERIOD)
self.sendMessageCommandMethod(bot, update,
"The maximum possible period is {0}.\nSetting period to {0}.".format(
str(MAX_PICTURE_SEND_PERIOD)))
# If a period length is fine - accept
else:
self.database_handler.setPeriod(chat_id, new_period)
self.sendMessageCommandMethod(bot, update,
"Setting period to {0}.".format(new_period)
)
# Reset timer
self.database_handler.resetTimer(chat_id)
self.restartPeriodicTask(chat_id)
return True
# user has sent a bullsh*t command
except ValueError:
return False
def doGimmepic(self, chat_id):
if self.pic_source == "local":
self.sendLocalRandomPic(chat_id)
elif self.pic_source == "DB":
self.sendDropboxRandomPic(chat_id)
def _periodicSender(self, bot, job):
chat_id = job.context
self.doGimmepic(chat_id)
self.database_handler.resetTimer(chat_id)
self.restartPeriodicTask(chat_id)
def restartPeriodicTask(self, chat_id):
self.removePeriodicSenderTask(chat_id)
self.createPeriodicSenderTask(chat_id)
def createPeriodicSenderTask(self, chat_id):
time_left = self.database_handler.getSendTime(chat_id) - time()
log.debug("Time left:", time_left)
job = Job(self._periodicSender, time_left, context=chat_id)
subscriptions_tasks[chat_id] = job
self.job_queue.put(job)
def removePeriodicSenderTask(self, chat_id):
subscriptions_tasks[chat_id].schedule_removal() # remove task from job queue
del subscriptions_tasks[chat_id]
##########
# COMMAND METHODS
##########
# GENERIC COMMANDS
# noinspection PyArgumentList
@_command_method
def command_start(self, bot, update, lS=None):
self.sendMessageCommandMethod(bot, update, lS(START_MESSAGE))
# noinspection PyArgumentList
@_command_method
def command_help(self, bot, update, lS=None):
msg = lS(HELP_MESSAGE).format(sr['picture_send_period'],MIN_PICTURE_SEND_PERIOD, MAX_PICTURE_SEND_PERIOD)
self.sendMessageCommandMethod(bot, update, msg)
# noinspection PyArgumentList
@_command_method
def command_about(self, bot, update, lS=None):
msg = lS(ABOUT_MESSAGE).format(".".join([str(i) for i in VERSION_NUMBER]))
self.sendMessageCommandMethod(bot, update, msg, disable_web_page_preview=False)
# noinspection PyArgumentList
# @_command_method
def command_otherbots(self, bot, update, lS=None):
# a = 2/0
self.sendMessageCommandMethod(bot, update, OTHER_BOTS_MESSAGE)
# noinspection PyArgumentList
@_command_method
def messageMethod(self, bot, update, lS=None):
chat_id = update.message.chat_id
message = update.message.text
log.info("messageMethod. Chat_id:", chat_id, "Message:", message)
if message in LanguageSupport.allVariants(HELP_BUTTON):
self.command_help(bot, update, lS)
elif message in LanguageSupport.allVariants(ABOUT_BUTTON):
self.command_about(bot, update, lS)
elif message in LanguageSupport.allVariants(OTHER_BOTS_BUTTON):
self.command_otherbots(bot, update, lS)
# elif message == EN_LANG_BUTTON:
# self.command_set_lang_en(bot, update, lS)
# elif message == RU_LANG_BUTTON:
# self.command_set_lang_ru(bot, update, lS)
elif message in LanguageSupport.allVariants(GIMMEPIC_BUTTON):
self.command_gimmepic(bot, update, lS)
elif message in LanguageSupport.allVariants(SUBSCRIBE_BUTTON):
self.command_subscribe(bot, update, lS)
elif message in LanguageSupport.allVariants(UNSUBSCRIBE_BUTTON):
self.command_unsubscribe(bot, update, lS)
elif message in LanguageSupport.allVariants(SHOW_PERIOD_BUTTON):
self.command_show_period(bot, update, lS)
else:
if not self.setPeriod(bot, update, lS):
self.unknown_command(bot, update, lS)
# noinspection PyArgumentList
@_command_method
def unknown_command(self, bot, update, lS=None):
self.sendMessageCommandMethod(bot, update, UNKNOWN_COMMAND_MESSAGE)
def error_handler(self, bot, update, error):
print(error)
# PICBOT COMMANDS
# noinspection PyArgumentList
@_command_method
def command_gimmepic(self, bot, update, lS=None):
chat_id = update.message.chat_id
self.doGimmepic(chat_id)
# noinspection PyArgumentList
@_command_method
def command_subscribe(self, bot, update, lS=None):
chat_id = update.message.chat_id
period = self.database_handler.getPeriod(chat_id)
if self.database_handler.getSubscribed(chat_id):
self.sendMessageCommandMethod(bot, update, lS(ALREADY_SUBSCRIBED_MESSAGE).format(period))
else:
self.database_handler.subscribeUser(chat_id)
self.database_handler.resetTimer(chat_id)
self.createPeriodicSenderTask(chat_id)
self.sendMessageCommandMethod(bot, update, lS(SUBSCRIBED_MESSAGE).format(period))
# noinspection PyArgumentList
@_command_method
def command_unsubscribe(self, bot, update, lS=None):
chat_id = update.message.chat_id
if not self.database_handler.getSubscribed(chat_id):
self.sendMessageCommandMethod(bot, update, lS(NOT_SUBSCRIBED_YET_MESSAGE))
else:
self.database_handler.unsubscribeUser(chat_id)
self.removePeriodicSenderTask(chat_id)
self.sendMessageCommandMethod(bot, update, lS(UNSUBSCRIBED_MESSAGE))
# noinspection PyArgumentList
@_command_method
def command_show_period(self, bot, update, lS=None):
chat_id = update.message.chat_id
period = self.database_handler.getPeriod(chat_id)
self.sendMessageCommandMethod(bot, update, """An image is sent to you every {0} seconds.""".format(period))
# noinspection PyArgumentList
@_command_method
def command_spamuncached(self, bot, update, lS=None):
chat_id = update.message.chat_id
self.sendUncachedImages(chat_id, self.pic_source)
def pullEquippedFileSystem(self, cloudService):
# initialize chosen session
if cloudService == 'drive':
cloudHandler = GoogleDriveHandler()
elif cloudService == 'dropbox':
# get access token if exists
accessToken = self.getSetting('dbAccessToken')
if accessToken is None:
print("> Initializing dropbox for first time")
cloudHandler = DropboxHandler(accessToken)
# update saved access code
self.setSetting('dbAccessToken', cloudHandler.access_token)
else:
print("> Error: Unrecognized cloud service")
return
# get UUIDs for equipped system
equippedSystem = self.getEquippedSystem()
if equippedSystem is None:
return
info = self.getSystemInfo(equippedSystem)
if info is None:
print("> Error: No info found for system '{}'".format(
equippedSystem))
filesInfo = info['files'] if 'files' in info else []
fileUUIDs = {}
for fileName, fInfo in filesInfo.items():
if 'uuid' in fInfo:
fileUUIDs[fileName] = fInfo['uuid']
if len(fileUUIDs) == 0:
print(
"> No encrypted files to pull! Encrypt the equipped filesystem using the 'encrypt' command and push it using 'push'"
)
return
# pull and timestamp (seconds) each file being pushed
for fileName, uuid in fileUUIDs.items():
# ensure last encryption was before it was pushed
if 'encrypted' not in filesInfo[fileName]:
print(
"> File system not yet encrypted! Encrypt it using 'encrypt' and push using 'push'."
)
return
if (filesInfo[fileName]['encrypted'] >=
filesInfo[fileName]['pushed']):
cfrm = input(
"> Looks like file '{}' was re-encrypted locally after it was pushed. Resync by pushing now?"
.format(fileName))
if str.lower(cfrm) != "yes":
print("> Operation aborted.")
return
self.pushEquippedFileSystem(cloudService)
return
# create crypt if doesn't exist
if not os.path.exists('crypt'):
os.mkdir('crypt')
# pull file
if cloudService == 'drive':
res = cloudHandler.download_file(uuid)
elif cloudService == 'dropbox':
res = cloudHandler.download_file("crypt/{}".format(uuid), uuid)
else:
print("> Error: Unrecognized cloud service")
return
if res is not None:
print("> Download successful.")
else:
print(" Download failed.")
return
# timestamp with pulled time
filesInfo[fileName]['pulled'] = round(time.time())
print(
"> Successfully '{}' ({}) from {}. Decrypt the filesystem to import it from the crypt."
.format(uuid, fileName, cloudService))
# update the database
systems = self.db.table('systems')
System = Query()
systems.update({'files': filesInfo}, System.name == equippedSystem)