def validate_fields(required_fields, request_json): try: for field in required_fields: if request_json.get(field) is None: raise APIBadRequest("{} is required".format(field)) except (ValueError, AttributeError): raise APIBadRequest(f"{' '.join(required_fields)} is/are required")
async def get_chats(request): skip = [request.args.get("skip"), 0][request.args.get("skip") == None] limit = [request.args.get("limit"), 10][request.args.get("limit") == None] start_date = request.args.get("start_date") end_date = request.args.get("end_date") username = request.args.get("username") if not username: raise APIBadRequest("username is required") search_text = request.args.get("search_text") if search_text: search_text = search_text.lower() logger.info(request.args) if not username: raise APIBadRequest("Username for this datasource is required") if start_date: start_date = dateparser.parse(start_date) if end_date: end_date = dateparser.parse(end_date) if start_date and end_date: if end_date < start_date: raise APIBadRequest("Start date should be less than End date") _chats, count = await filter_chats( request.app.config[DATASOURCE_NAME]["tables"]["chat_table"], username, start_date, end_date, int(skip), int(limit), search_text) for chat in _chats: chat.pop("message_content") chat.pop("chat_path") if chat.get("messages"): messages = json.loads(chat.get("messages")) chat.update({"messages": messages}) ##we need to give one message to frontend, We need to loop over messages till the time ## we get a message which is not None, then break the loop for message in messages: if message.get("content"): chat.update({"last_message": message.get("content")}) break if chat.get("participants"): participants = json.loads(chat.get("participants")) chat.update({"participants": participants}) return response.json({ 'error': False, 'success': True, "message": None, "data": { "chats": _chats, "count": count } })
def extract(src_path: str, dst_path_prefix: str, config: Dict[str, Any], datasource_name: str, username: str) -> Tuple[str, str]: """ src_path : where the user has downloaded their ZIP file, temp_directory = tempfile.TemporaryDirectory() """ # temp_directory = tempfile.mkdtemp() logger.info("Entered into the extract function") if not os.path.exists(src_path): raise APIBadRequest("This path doesnt exists") # try: # the_zip_file = zipfile.ZipFile(src_path) # except: # raise APIBadRequest("Invalid zip file") # logger.info(f"Testing zip {src_path} file") # ret = the_zip_file.testzip() # if ret is not None: # raise APIBadRequest(f"Invalid zip datasource_name file") _checksum = checksum(src_path) archives_present = get_archives( config[datasource_name]["tables"]["archives_table"], _checksum) if archives_present: raise APIBadRequest("Zip file already have been uploaded") utc_timestamp = datetime.datetime.utcnow().strftime("%d-%m-%Y") dst_path_suffix = f"{utc_timestamp}-{_checksum}" logger.info(f"This is the new destination suffix {dst_path_suffix}") dst_path = os.path.join(dst_path_prefix, username, dst_path_suffix) try: with zipfile.ZipFile(src_path) as zf: zf.extractall(dst_path) #shutil.unpack_archive(src_path, extract_dir=dst_path, format=None) except MemoryError: logger.error( f"We ran out of memory while processing {datasource_name}, Please try again" ) raise Exception( f"We ran out of memory while processing {datasource_name}, Please try again" ) except: raise APIBadRequest(f"Invalid zip {datasource_name} file") logger.info(f"Setting new archival for {datasource_name} ") set_archives(config[datasource_name]["tables"]["archives_table"], dst_path, username, _checksum) logger.info(f"This is the dst_path for {datasource_name} is {dst_path}") return _checksum, dst_path
async def aws_temp_creds(self): creds = await get_credentials( self.config[USER_DATASOURCE_NAME]["tables"]["creds_table"]) if not creds: raise APIBadRequest("User is not logged in") creds = list(creds)[0] r = requests.post(self.config.LOGIN, data=json.dumps({ "username": creds["username"], "password": creds["password"] })) result = r.json() if result.get("error"): logger.error(result["message"]) raise APIBadRequest(result["message"]) r = requests.post( self.config.TEMPORARY_S3_CREDS, data=json.dumps({"id_token": result["data"]["id_token"]}), headers={"Authorization": result["data"]["id_token"]}) result = r.json() if result.get("error"): logger.error(result["message"]) raise APIBadRequest(result["message"]) return result["data"]["identity_id"], result["data"][ "access_key"], result["data"]["secret_key"], result["data"][ "session_token"]
async def github_re_backup_whole(request): """ """ username = request.args.get("username") if not username: raise APIBadRequest("Username for this datasource is required") try: username, password = await get_creds( request.app.config[DATASOURCE_NAME]["tables"]["creds_table"], request.json["username"]) except: raise APIBadRequest("Credentials aren't present") #update_datasources_status(request.app.config.DATASOURCES_TBL , "CODEREPOS/Github",request.json["username"] , request.app.config.DATASOURCES_CODE["REPOSITORY"]["GITHUB"], "IN_PROGRESS", "PROGRESS") # else: # raise APIBadRequest("Unknown format") # logger.info(f"THe request was successful with github path {request.json['path']}") request.app.add_task( background_github_parse(request.app.config, username, password, re_backup=True)) return response.json({ 'error': False, 'success': True, "message": "Your whole github backup has started, Once done it will start reflecting on your github Dashboard", "data": None, })
async def store_permissions(request): """ {'backup': ['stats_table', 'status_table', 'backup_table', 'backup_list_table'], 'coderepos': ['creds_table', 'stats_table', 'status_table', 'archives_table', 'repos_table'], 'facebook': ['creds_table', 'image_table', 'archives_table', 'yourposts_table', 'other_posts', 'content', 'chat_table', 'chat_content', 'stats_table', 'address_table', 'status_table'], 'google': ['creds_table', 'image_table', 'email_table', 'email_attachment_table', 'email_content_table', 'purchase_table', 'archives_table', 'reservation_table', 'location_table', 'location_approximate_table', 'stats_table', 'status_table'], 'twitter': ['creds_table', 'archives_table', 'stats_table', 'status_table', 'account_table', 'tweet_table', 'indexed_tweet_table'], 'users': ['creds_table']} """ request.app.config.VALIDATE_FIELDS(["plugin_name", "permissions"], request.json) logger.debug(request.json["permissions"]) if not isinstance(request.json['permissions'], list): raise APIBadRequest("Permissions should be instance of list") if not request.json["plugin_name"] in request.app.config.plugins: raise APIBadRequest("Plugin is unknown") for permission in request.json["permissions"]: permission.update({"tbl_object": permission_table, "plugin_name": request.json["plugin_name"] }) await store_permission(**permission) return response.json({ 'error': False, 'success': True, "message": None, "data": None})
async def check_mnemonic(request): """ API to be used when user has reinstalled datapod since he/she already has menmonic intialized somewhere in the past , this mnemonic has to be checked against the hash of the Mnemonic """ request.app.config.VALIDATE_FIELDS(["mnemonic"], request.json) creds = await get_credentials( request.app.config[USER_DATASOURCE_NAME]["tables"]["creds_table"]) if not creds: raise APIBadRequest("User is not logged in") creds = list(creds)[0] r = requests.post(request.app.config.LOGIN, data=json.dumps({ "username": creds["username"], "password": creds["password"] })) login_result = r.json()["data"] ##converting mnemonic list of words into a string of 24 words of mnemonic mnemonic = " ".join(request.json["mnemonic"]) logger.debug(f"The mnemonic is {mnemonic}") sha3_256 = hashlib.sha3_256(mnemonic.encode()).hexdigest() logger.debug(f"Sha 256 of mnemonic is {sha3_256}") r = requests.post(request.app.config.CHECK_MNEMONIC, data=json.dumps({ "username": login_result["username"], "sha3_256": sha3_256, }), headers={"Authorization": login_result["id_token"]}) check_mnemonic_result = r.json() if check_mnemonic_result["error"]: raise APIBadRequest(check_mnemonic_result["message"]) ##TODO Store mnemonic in the local db mnemonic_keys = child_keys(mnemonic, 0) await update_mnemonic_n_address( request.app.config[USER_DATASOURCE_NAME]["tables"]["creds_table"], login_result["username"], mnemonic, mnemonic_keys["address"], mnemonic_keys["private_key"]) return response.json({ "error": False, "success": True, "message": "Mnemonic has been saved on your machine", "data": None })
async def tweets(request): logger.info("Number is ", request.args.get("limit")) skip = [request.args.get("skip"), 0][request.args.get("skip") == None] limit = [request.args.get("limit"), 10][request.args.get("limit") == None] matching_string = request.args.get("match_string") start_date = request.args.get("start_date") end_date = request.args.get("end_date") username = request.args.get("username") if not username: raise APIBadRequest("Username for this datasource is required") logger.info(f"Params are {request.args}") if start_date: start_date = dateparser.parse(start_date) if end_date: end_date = dateparser.parse(end_date) if start_date and end_date: if end_date < start_date: raise APIBadRequest("Start date should be less than End date") logger.info(f"This is the start_date {start_date}") logger.info(f"This is the end_date {end_date}") if matching_string: logger.info(f"THis is the matchiing_String {matching_string}") result, count = await match_text(request.app.config[DATASOURCE_NAME]["tables"]["tweet_table"], username, request.app.config[DATASOURCE_NAME]["tables"]["indexed_tweet_table"], \ matching_string , start_date, end_date, int(skip), int(limit)) else: result, count = await filter_tweet( request.app.config[DATASOURCE_NAME]["tables"]["tweet_table"], username, start_date, end_date, int(skip), int(limit)) # [repo.update({ # "created_at":repo.get("created_at").strftime("%d, %b %Y"), # }) for repo in result] return response.json({ 'error': False, 'success': True, 'data': { "tweets": result, "count": count }, 'message': None })
async def attachment_filter(request): """ To get all the assets created by the requester """ logger.debug(f"Args are {request.args.items()}") skip = [request.args.get("skip"), 0][request.args.get("skip") == None] limit = [request.args.get("limit"), 50][request.args.get("limit") == None] start_date = request.args.get("start_date") end_date = request.args.get("end_date") message_type = request.args.get("message_type") matching_string = request.args.get("match_string") username = request.args.get("username") if not username: raise APIBadRequest("Username for this datasource is required") # logger.debug(f"Skip type is {skip}") # logger.debug(f"limit type is {limit}") # logger.debug(f"start date type is {start_date}, and type is {type(start_date)}") if start_date: start_date = dateparser.parse(start_date) if end_date: end_date = dateparser.parse(end_date) if start_date and end_date: if end_date < start_date: raise APIBadRequest("Start date should be less than End date") if not matching_string: attachments, count = await filter_attachments( request.app.config[DATASOURCE_NAME]["tables"] ["email_attachment_table"], username, message_type, start_date, end_date, int(skip), int(limit)) else: attachments, count = await filter_attachments_on_text( request.app.config[DATASOURCE_NAME]["tables"] ["email_attachment_table"], username, message_type, start_date, end_date, int(skip), int(limit), matching_string) # for iage in images: # creation_time = image.pop("creation_time") # #data:image/png;base64 # image.update({"creation_time": creation_time.strftime("%Y-%m-%d")}) return response.json({ 'error': False, 'success': True, "data": { "result": attachments, "count": count }, "message": None })
async def image_filter(request): logger.debug("Number is ", request.args.get("limit")) skip = [request.args.get("skip"), 0][request.args.get("skip") == None] limit = [request.args.get("limit"), 10][request.args.get("limit") == None] start_date = request.args.get("start_date") end_date = request.args.get("end_date") username = request.args.get("username") if not username: raise APIBadRequest("Username for this datasource is required") logger.debug(f"Params are {request.args}") if start_date: start_date = dateparser.parse(start_date) if end_date: end_date = dateparser.parse(end_date) if start_date and end_date: if end_date < start_date: raise APIBadRequest("Start date should be less than End date") images, count = await filter_images( request.app.config[DATASOURCE_NAME]["tables"]["image_table"], username, start_date, end_date, int(skip), int(limit)) logger.debug(images) for image in images: b64_data = await image_base64(image['image_path']) creation_time = image.pop("creation_time") encoded_string = "data:image/jpeg;base64," + b64_data #data:image/png;base64 image.update({ "creation_time": creation_time.strftime("%Y-%m-%d"), "uri": encoded_string }) # [repo.update({ # "created_at":repo.get("created_at").strftime("%d, %b %Y"), # }) for repo in result] return response.json({ 'error': False, 'success': True, 'data': { "images": images, "count": count }, 'message': None })
async def parse(request): """ To get all the assets created by the requester """ request.app.config.VALIDATE_FIELDS(["path", "username"], request.json) res = await get_status( request.app.config[DATASOURCE_NAME]["tables"]["status_table"]) res = list(res) logger.info(res) if res: for element in res: if element.get("status") == "PROGRESS": raise APIBadRequest( "Already processing a twitter for the user") username = request.json["username"].lower() config = request.app.config dst_path_prefix = os.path.join(config.RAW_DATA_PATH, DATASOURCE_NAME) logger.info( f"The dst_path_prefix fo rthis datasource is {dst_path_prefix}") try: checksum, dest_path = await extract(request.json["path"], dst_path_prefix, config, DATASOURCE_NAME, username) await update_status(config[DATASOURCE_NAME]["tables"]["status_table"], DATASOURCE_NAME, username, "PROGRESS", checksum, dest_path, request.json["path"]) except Exception as e: logger.error(e) await delete_status(config[DATASOURCE_NAME]["tables"]["status_table"], DATASOURCE_NAME, username) raise APIBadRequest(e) request.app.add_task( _parse(request.app.config, dest_path, username, checksum)) return response.json({ 'error': False, 'success': True, "message": "Twitter parsing has been Started and you will be notified once it is complete", "data": None })
async def decorated_function(request, *args, **kwargs): # run some method that checks the request # for the client's authorization status #is_authorized = check_request_for_authorization_status(request) result = await get_credentials(request.app.config["Users"]["tables"]["creds_table"]) #logger.info(f"Data from the credential table in id_token_validity decorator {result}") if not result: logger.error("Credentials aren't present, Please Login again") raise APIBadRequest("Credentials aren't present, Please Login again") result = list(result)[0] try: id_token = convert_type(result["id_token"]) access_token = convert_type(result["access_token"]) refresh_token = convert_type(result["refresh_token"]) username = result["username"] ##this is because all the token are byte object, we need to upate user object in request object # with str type of tokens result.update({"id_token": id_token, "access_token": access_token, "refresh_token": refresh_token}) request["user"] = result except Exception as e: logger.error(f"User must have signed out, Please Login again with an error {e.__str__()}") raise APIBadRequest("Please Login again") payload = jwt.get_unverified_claims(id_token) time_now = datetime.datetime.fromtimestamp(revoke_time_stamp(timezone=request.app.config.TIMEZONE)) time_expiry = datetime.datetime.fromtimestamp(payload["exp"]) rd = dateutil.relativedelta.relativedelta (time_expiry, time_now) logger.warning("Difference between now and expiry of id_token") logger.warning(f"{rd.years} years, {rd.months} months, {rd.days} days, {rd.hours} hours, {rd.minutes} minutes and {rd.seconds} seconds") if rd.minutes < 20: logger.error("Renewing id_token, as it will expire soon") id_token = update_tokens(request.app.config, username, refresh_token) if isinstance(id_token, bytes): id_token = id_token.decode() response = await f(request, *args, **kwargs) return response
async def create(self, src_path, dst_path, datasource_name): #temp = tempfile.NamedTemporaryFile('wb', suffix='.tar.lzma', delete=False) temp = tempfile.NamedTemporaryFile('wb', suffix='.tar.gz', delete=False) #temp = tempfile.TemporaryFile() # backup_path = f"{self.backup_path}/{archival_name}/backup.tar.lzma" ##this is the file under ~/.datapod/user_indexes for a corresponding datasource ## which wil keep track of all the files which have been backed up previously user_index_file = os.path.join(self.user_index_dir, f"{datasource_name.lower()}.index") logger.debug( f"{datasource_name} This is the user_index_file {user_index_file}, used to create a compressed file at {temp.name} from a directory at {src_path} " ) if platform.system() == "Linux": if self.full_backup: backup_command = f"tar --create --gzip --no-check-device --verbose -f {temp.name} {src_path}" else: backup_command = f"tar --create --gzip --no-check-device --verbose --listed-incremental={user_index_file} -f {temp.name} {src_path}" elif platform.system() == "Darwin": if self.full_backup: backup_command = f"gtar --create --lzma --no-check-device --verbose -f {temp.name} {src_path}" else: backup_command = f"gtar --create --lzma --no-check-device --verbose --listed-incremental={user_index_file} -f {temp.name} {src_path}" else: raise APIBadRequest( "The platform is not available for this os distribution") #backup_command = f"tar --create --verbose --listed-incremental={user_index_file} --lzma {backup_path} {self.raw_data_path}" initial_time = int(time.time()) next_time = initial_time + 15 for out in self.config.OS_COMMAND_OUTPUT(backup_command, "Backup"): if int(time.time()) >= next_time: # await self.send_sse_message(f"Archiving {out.split('/')[-1]} for {datasource_name}") logger.debug( f"Archiving {out.split('/')[-1]} for {datasource_name}") next_time += 10 split_backup_dir = tempfile.mkdtemp() logger.debug( f"Now, splitting the single compressed file {temp.name} in a temporary directory {split_backup_dir}" ) async for msg in self.split(split_backup_dir, temp.name): # await self.send_sse_message(msg) logger.debug(msg) ##because temp.name will automatically be removed logger.debug(f"Now removing single comporessed file at {temp.name}") self.remove_temporary_archive(temp.name) return split_backup_dir
async def dashboard(request): username = request.args.get("username") if not username: raise APIBadRequest("Username for this datasource is required") res = await get_account( request.app.config[DATASOURCE_NAME]["tables"]["account_table"], username) result = res[0] result.update({ "common_hashtags": json.loads(result["common_hashtags"]), "common_user_mentions": json.loads(result["common_user_mentions"]) }) return response.json({ 'error': False, 'success': True, 'data': result, "message": None })
async def __init__(self, config): """ number is the percentage number which will be sent in sse message, Then number has already been incremented by the backup scripts above, """ self.config = config self.bucket_name = config.AWS_S3['bucket_name'] #self.credentials = get_credentials(config.CREDENTIALS_TBL) self.credentials = await get_credentials( self.config[USER_DATASOURCE_NAME]["tables"]["creds_table"]) if not self.credentials: raise APIBadRequest("User is not logged in") self.credentials = list(self.credentials)[0] ##in this temporary file, private key is now written if not self.credentials["encryption_key"]: raise MnemonicRequiredError() self.encryption_key = binascii.unhexlify( self.credentials["encryption_key"].encode()) self.identity_id, self.access_key, self.secret_key, self.session_token = await self.aws_temp_creds( ) os.environ[ 'AWS_ACCESS_KEY_ID'] = self.access_key # visible in this process + all children os.environ[ 'AWS_SECRET_ACCESS_KEY'] = self.secret_key # visible in this process + all children os.environ[ 'AWS_SESSION_TOKEN'] = self.session_token # visible in this process + all children os.environ["AWS_DEFAULT_REGION"] = self.config.AWS_S3["default_region"]
def store_credentials(credential_table, username, password, password_hash, id_token, access_token, refresh_token, name, email): query = credential_table.select() usernames = [user.username for user in query] print (usernames) if usernames: if username not in usernames: raise Exception("Different usernames arent allowed on the same machine") try: credential_table.insert(username=username, password_hash=password_hash, password=password, id_token=id_token, access_token= access_token, refresh_token=refresh_token, name = name, email=email ).execute() except IntegrityError: logger.info(f"Credentials for the user already exists, updating it now") credential_table.update( id_token=id_token, access_token= access_token, refresh_token=refresh_token)\ .where(credential_table.username==username).\ execute() except Exception as e: logger.error("Saving credentials of the users failed {e}") raise APIBadRequest("Could save credentials because of {e.__str__()}") return
async def start_fresh_backup(request): """ ##TODO ADD entries to BACKUP_TBL """ ##This has a rare chance of happening, that users reach here and doesnt have mnemonic in the database but have mnemonic in the cloud def str2bool(v): return v.lower() in ("yes", "true", "t", "1") full_backup = request.args.get("full_backup") if not full_backup: raise APIBadRequest( "Please specify which kind of backup is required, for full backup send True, for incremental_backup send False" ) full_backup = str2bool(full_backup) if not isinstance(full_backup, bool): raise APIBadRequest("full_backup should only be a bool type") await update_status( request.app.config[DATASOURCE_NAME]["tables"]["status_table"], DATASOURCE_NAME, "backup", "PROGRESS") ##this is being called here, because this will try to fetch temporary creds from AWS from the id_token stored in the creds user ##table, if they are stale, it will send an error to the user directly to login again, ##TODO, find a better way to to this backup_instance = await S3Backup(request.app.config) try: request.app.add_task( backup_upload(request.app.config, full_backup, backup_instance)) # new_log_entry = request.app.config.LOGS_TBL.create(timestamp=archival_object, message=f"Archival was successful on {archival_name}", error=0, success=1) # new_log_entry.save() except Exception as e: logger.error(e.__str__()) # new_log_entry = request.app.config.LOGS_TBL.create(timestamp=archival_object, message=f"Archival failed because of {e.__str__()} on {archival_name}", error=1, success=0) # new_log_entry.save() return response.json({ 'error': False, 'success': True, })
async def decorated_function(request, *args, **kwargs): # run some method that checks the request # for the client's authorization status #is_authorized = check_request_for_authorization_status(request) if not request.app.config.TESTING_MODE: raise APIBadRequest("This API cant be executed in Production environment") response = await f(request, *args, **kwargs) return response
async def delete_original_path(request): """ After the processing of the whole data source, this api can be used to delete the original zip correspoding to a particular username """ username = request.args.get("username") if not username: raise APIBadRequest("Username for this datasource is required") res = await get_status( request.app.config[DATASOURCE_NAME]["tables"]["status_table"], username) result = list(res) logger.debug(result[0].get("username")) if not result: raise APIBadRequest( f"No status present for {DATASOURCE_NAME} for username {username}") result = result[0] logger.debug(result) path_to_be_deleted = result.get("original_path") logger.warning(f"Path to be deleted is {path_to_be_deleted}") try: os.remove(path_to_be_deleted) logger.success(f"{path_to_be_deleted} is deleted now") except Exception as e: return response.json({ 'error': False, 'success': True, "message": f"Original path at {path_to_be_deleted} couldnt be delete because of {e.__str__()}", "data": None }) return response.json({ 'error': False, 'success': True, "message": f"Original path at {path_to_be_deleted} is deleted", "data": None })
async def purchases_filter(request): skip = [request.args.get("skip"), 0][request.args.get("skip") == None] limit = [request.args.get("limit"), 10][request.args.get("limit") == None] start_date = request.args.get("start_date") end_date = request.args.get("end_date") merchant_name = request.args.get("match_string") username = request.args.get("username") if not username: raise APIBadRequest("Username for this datasource is required") logger.debug(f"Params are {request.args}") if start_date: start_date = dateparser.parse(start_date) if end_date: end_date = dateparser.parse(end_date) if start_date and end_date: if end_date < start_date: raise APIBadRequest("Start date should be less than End date") # logger.debug(f"This is the start_date {start_date}") # logger.debug(f"This is the end_date {end_date}") purchases, count = await filter_purchases( request.app.config[DATASOURCE_NAME]["tables"]["purchase_table"], username, start_date, end_date, int(skip), int(limit), merchant_name) result = [ format_purchase(request.app.config, purchase) for purchase in purchases ] return response.json({ 'error': False, 'success': True, 'data': { "purchases": result, "count": count }, 'message': None })
async def make_backup(self): """ --level=0, for fullbackup --level=1, for incremental backup --listed_incremental is equivalent to -g --atime-preserve=system brew install gnu-tar #tar --create --lzma --verbose --multi-volume --tape-length 102400 --file=MyArchive.tgz raw -g user.index With --newer you're simply updating/creating the archive with the files that have changed since the date you pass it. tar --create --lzma --verbose --file=MyArchive raw/facebook/facebook-sauravverma14473426 """ datasources = os.listdir(self.raw_data_path) logger.debug(datasources) if not datasources: raise APIBadRequest( "The directory whose backup needs to be made is empty") archival_object = datetime.datetime.utcnow() archival_name = archival_object.strftime("%B-%d-%Y_%H-%M-%S") parent_destination_path = os.path.join(self.backup_path, archival_name) s3_backup_instance = await BotoBackup(self.config) step = int(90 / len(datasources)) for (index, datasource_name) in enumerate(datasources): s3_folder_name = archival_name + "/" + datasource_name dst_path = os.path.join(self.backup_path, archival_name, datasource_name) src_path = os.path.join(self.raw_data_path, datasource_name) # if not os.path.exists(dst_path): # os.makedirs(dst_path) backup_archival_temporary_path = await self.create( src_path, dst_path, datasource_name) # # res = {"message": "Progress", "percentage": int(i*step)} # # await self.config["send_sse_message"](config, DATASOURCE_NAME, res) await s3_backup_instance.sync_backup( datasource_name, backup_archival_temporary_path, archival_name) ##the split archival for a datasource in a temporary folder hasnt been removed yet, removing it now self.remove_split_archival_dir(backup_archival_temporary_path) logger.debug( f"Now removing the split files present {backup_archival_temporary_path} " ) self.percentage = (index + 1) * step await self.send_sse_message( f"Archiving of {datasource_name} completed") self.percentage = 100 await self.send_sse_message(f"Backup completed") return parent_destination_path, archival_name
async def single_chat(request): """ To get all the chats created by the user thread_path = 'inbox/KapilDevGarg_lapjbN90Hw' """ thread_path = request.args.get("thread_path") if not thread_path: raise APIBadRequest("thread_path is required") logger.info(f'This is the chat id {request.args.get("chat_id")}') ds_path = os.path.join(request.app.config.RAW_DATA_PATH, f"facebook/messages/{thread_path}") if not os.path.exists(ds_path): raise APIBadRequest("This thread_path doesnt exists") logger.info(ds_path) chats = [] chat_files = [(os.path.join(ds_path, file)) for file in os.listdir(ds_path)] for _file in chat_files: with open(_file, "r") as json_file: data = json.load(json_file) logger.info(data) chats.extend(data.get("messages")) # if request.json.get("message_type") not in chat_types: # raise APIBadRequest("This message type is not available") # chat_path = os.path.join(ds_path, request.json.get("message_type")) # all_chats = os.listdir(chat_path) # result = [{"name": e.split("_")[0], "chat_id": e} for e in all_chats] return response.json({ 'error': False, 'success': True, "message": "Facebook data parsing has been Started and you will be notified once it is complete", "data": chats })
async def locations_filter(request): logger.debug(f"Args are {request.args.items()}") start_date = request.args.get("start_date") end_date = request.args.get("end_date") username = request.args.get("username") if not username: raise APIBadRequest("Username for this datasource is required") logger.debug(f"Params are {request.args}") if start_date: start_date = dateparser.parse(start_date) start_date = calendar.timegm(start_date.timetuple()) if end_date: end_date = dateparser.parse(end_date) end_date = calendar.timegm(end_date.timetuple()) if start_date and end_date: if end_date < start_date: raise APIBadRequest("Start date should be less than End date") locations, count = await filter_locations( request.app.config[DATASOURCE_NAME]["tables"] ["location_approximate_table"], username, start_date, end_date) logger.debug(locations) #result = [format_purchase(request.app.config, purchase) for purchase in purchases] result = list(locations) return response.json({ 'error': False, 'success': True, 'data': { "locations": result, "count": count }, 'message': None })
def revoke_time_stamp(days=0, hours=0, minutes=0, timezone=None): if not timezone: logger.error("Please specify valid timezone for your servers") raise APIBadRequest("Please specify valid timezone for your servers") tz_kolkata = pytz.timezone(timezone) time_format = "%Y-%m-%d %H:%M:%S" naive_timestamp = datetime.datetime.now() aware_timestamp = tz_kolkata.localize(naive_timestamp) ##This actually creates a new instance od datetime with Days and hours _future = datetime.timedelta(days=days, hours=hours, minutes=minutes) result = aware_timestamp + _future return result.timestamp()
async def login(config): creds = await get_credentials( config[USER_DATASOURCE_NAME]["tables"]["creds_table"]) if not creds: raise APIBadRequest("User is not logged in") creds = list(creds)[0] #r = requests.post(request.app.config.LOGIN, data=json.dumps({"username": creds["username"], "password": creds["password"]})) if creds["mnemonic"]: raise APIBadRequest("The mnemonic is already present") ##renew tokens r = requests.post(config.LOGIN, data=json.dumps({ "username": creds["username"], "password": creds["password"] })) login_result = r.json() return login_result["data"]
async def aws_temp_creds(self): creds = await get_credentials( self.config[USER_DATASOURCE_NAME]["tables"]["creds_table"]) if not creds: raise APIBadRequest("User is not logged in") creds = list(creds)[0] # r = requests.post(self.config.LOGIN, data=json.dumps({"username": creds["username"], "password": creds["password"]})) # result = r.json() # if result.get("error"): # logger.error(result["message"]) # raise APIBadRequest(result["message"]) r = requests.post( self.config.TEMPORARY_S3_CREDS, data=json.dumps({"id_token": creds["id_token"].decode()}), headers={"Authorization": creds["id_token"].decode()}) result = r.json() if result.get("message") == 'The incoming token has expired': return response.json( { "error": True, "sucess": False, "message": "The id token has expired, Please login again", "data": None }, status=401) if result.get("error"): logger.error(result["message"]) raise APIBadRequest(result["message"]) return result["data"]["identity_id"], result["data"][ "access_key"], result["data"]["secret_key"], result["data"][ "session_token"]
async def github_backup_single_repo(request): """ """ if not request.args.get("name"): raise APIBadRequest("Name of the repository is required") username = request.args.get("username") if not username: raise APIBadRequest("Username for this datasource is required") logger.info(request.args) result = await get_single_repository( request.app.config[DATASOURCE_NAME]["tables"]["repos_table"], username, request.args.get("name")) if not result: raise APIBadRequest("No repo exists") for repository in result: logger.info(repository) owner = json.loads(repository["owner"]) repository.update({"owner": owner}) request.app.add_task( per_repository(username, repository["path"], repository, request.app.config, None)) ##await per_repository(repository["path"], repository, request.app.config, None) return response.json({ 'error': False, 'success': True, 'message': f"Backup the repository {request.args.get('name')} has been started", 'data': None })
async def restart_parse(request): request.app.config.VALIDATE_FIELDS(["username"], request.json) res = await get_status( request.app.config[DATASOURCE_NAME]["tables"]["status_table"], request.json["username"]) result = list(res) if not result: raise APIBadRequest( f"No status present for {DATASOURCE_NAME} for username {request.json['username']}" ) result = result[0] original_path = result.get("original_path") if not original_path: raise APIBadRequest( f"No Path is present for {DATASOURCE_NAME} for username {request.json['username']}, Please cancel this processing" ) if not os.path.exists(original_path): raise APIBadRequest( f"This path {original_path} doesnts exists anymore, Please cancel this processing" ) request.app.add_task( start_parse(request.app.config, original_path, request.json["username"])) return response.json({ 'error': False, 'success': True, "message": "Takeout data parsing for {request.json['username']} has been restarted and you will be notified once it is complete", "data": None })
async def dashboard(request): username = request.args.get("username") if not username: raise APIBadRequest("username is required") res = await dashboard_data( username, request.app.config[DATASOURCE_NAME]["tables"]["image_table"], request.app.config[DATASOURCE_NAME]["tables"]["chat_table"], request.app.config[DATASOURCE_NAME]["tables"]["address_table"]) return response.json({ 'error': False, 'success': True, "message": None, "data": res })
async def decorated_function(request, *args, **kwargs): # run some method that checks the request # for the client's authorization status #is_authorized = check_request_for_authorization_status(request) result = get_credentials(request.app.config.CREDENTIALS_TBL) logger.info(f"Data from the credential table in id_token_validity decorator {result}") if not result: logger.error("Credentials aren't present, Please Login again") raise APIBadRequest("Credentials aren't present, Please Login again") username = result["username"] if isinstance(username, bytes): username = username.decode() response = await f(request, username, *args, **kwargs) return response