async def create_job(identifier, **data) -> Result:
server_config = data.pop("server_config", {})
link = data.pop("link", None) or settings.SCHEDULER_SPREADSHEET
sheet = data.pop("sheet", None) or settings.SCHEDULER_SHEET_NAME
single_job = data.pop("job", None)
multiple_jobs = data.pop("jobs", [])
endpoint = data.pop("endpoint", None)
method = data.pop("method", None)
if server_config:
config = server_config
else:
config = await get_provider_sheet(link=link, sheet=sheet, provider=identifier)
if config and all([endpoint, method, any([single_job, multiple_jobs])]):
instance = RPCService(config["host"], config["port"], config["server_method"])
if multiple_jobs:
job_ids = await asyncio.gather(
*[
instance.rpc_create_job(
args=[endpoint, method, json.dumps(x)],
**data,
)
for x in multiple_jobs
]
)
else:
job_id = await instance.rpc_create_job(
args=[endpoint, method, json.dumps(single_job)], **data
)
job_ids = [job_id]
return Result(data=job_ids)
return Result(
error="Error scheduling job, missing field, job, jobs,endpoint,method,"
)
async def get_authorization_url(provider,
link=None,
sheet=None,
key="client",
credentials=None,
**kwargs):
custom_credentials = {}
if credentials:
custom_credentials = credentials
provider_instance = await get_provider_sheet(link,
sheet,
provider,
key=key,
**custom_credentials)
if not provider_instance:
return Result(error="Could not find provider in link provided")
client_id = provider_instance.get("client_id")
if "client_id" in kwargs:
client_id = kwargs.pop("client_id")
if not client_id:
return Result(error="client_id not provided")
oauth_library = oauth_implementation(provider_instance,
client_id=client_id,
**kwargs)
if not oauth_library:
return Result(
error="Could not generate authorization url for provider passed")
authorization_url = resolve_authorization_url(oauth_library,
provider_instance, **kwargs)
if not authorization_url:
return Result(
error="Could not generate authorization url for provider passed")
return Result(data={"authorization_url": authorization_url})
async def fetch_groups(link, sheet, segments):
if not link or not sheet:
return Result(error="Missing `link` or `sheet` value")
instance = models.GoogleSheetInterface(**config)
instance.load_file(link, sheet)
result = instance.fetch_groups(segments)
return Result(data=result)
async def delete_job(identifier, **data) -> Result:
server_config = data.pop("server_config", {})
link = data.pop("link", None) or settings.SCHEDULER_SPREADSHEET
sheet = data.pop("sheet", None) or settings.SCHEDULER_SHEET_NAME
single_job = data.pop("job_id", None)
multiple_jobs = data.pop("job_ids", [])
if server_config:
config = server_config
else:
config = await get_provider_sheet(link=link, sheet=sheet, provider=identifier)
if config and any([single_job, multiple_jobs]):
instance = RPCService(config["host"], config["port"], config["server_method"])
if multiple_jobs:
await asyncio.gather(
*[instance.rpc_delete_job(job_id=x) for x in multiple_jobs]
)
else:
await instance.rpc_delete_job(job_id=single_job)
return Result(
data={
"status": "successful",
"job_id": single_job,
"job_ids": multiple_jobs,
}
)
return Result(error="Invalid parameter passed, job_id, job_ids")
async def similarity_check_api(**data) -> Result:
config = data.pop("server_config", {})
similarity_config = config.get("similarity") or {}
url = similarity_config.get("url")
api_key = similarity_config.get("api_key")
main_text = data.get("text")
other_texts = data.get("other_texts") or []
if url and api_key and main_text and other_texts:
def callback(o):
rr = requests.get(
url,
params={
"text1": o["text1"],
"text2": o["text2"],
"token": api_key,
"bow": "always",
},
)
if rr.status_code < 400:
return rr.json()
rr.raise_for_status()
results = await asyncio.gather(*[
loop_helper(lambda: callback({
"text1": main_text,
"text2": x
})) for x in other_texts
])
return Result(data=results)
return Result(error="Missing url, main_text, other_text or api_key")
async def read_last_row(link, sheet) -> Result:
if not link or not sheet:
return Result(error="Missing `link` or `sheet` value")
instance = models.GoogleSheetInterface(**config)
instance.load_file(link, sheet)
result = instance.read_last_row()
return Result(data=result)
async def read_sheetnames(link) -> Result:
if not link:
return Result(error="Missing `link` or `sheet` value")
instance = models.GoogleSheetInterface(**config)
result = instance.get_sheet_names(link)
title = instance.get_spreadsheet_title(link)
return Result(data=dict(title=title, sheet_names=result))
async def process_spell_check(**data):
web_spell_check, similarity_check = await asyncio.gather(
web_spell_check_api(**data), similarity_check_api(**data))
if web_spell_check.error or similarity_check.error:
return Result(
error={
"spelling": web_spell_check.error,
"similarity": similarity_check.error,
})
return Result(data={
"spelling": web_spell_check.data,
"similarity": similarity_check.data
})
async def read_referenced_cell(link, sheet, options, key, value) -> Result:
if not link or not sheet:
return Result(error="Missing `link` or `sheet` value")
instance = models.GoogleSheetInterface(**config)
instance.load_file(link, sheet)
result = instance.get_referenced_cell_values(options)
if value:
if not key:
return Result(error="Missing `key` field to read a single record")
found = [x for x in result if x[key] == value]
if found:
return Result(data=found[0])
return Result(data=result)
async def update_row(link, sheet, key, value, data) -> Result:
if not link or not sheet:
return Result(error="Missing `link` or `sheet` value")
instance = models.GoogleSheetInterface(**config)
instance.load_file(link, sheet)
if value:
if not key:
return Result(error="Missing `key` field to read a record")
try:
instance.update_existing_record(key, value, data)
except KeyError as e:
return Result(error="Wrong `key` passed in `data`")
else:
return await read_row(link, sheet, key, value)
async def read_new_row(link, sheet, page, page_size, key, value) -> Result:
if not link or not sheet:
return Result(error="Missing `link` or `sheet` value")
instance = models.GoogleSheetInterface(**config)
instance.load_file(link, sheet)
full_result = instance.get_all_records()
result = models.get_page(full_result, page_size=page_size, page=page)
if value:
if not key:
return Result(error="Missing `key` field to read a single record")
found = [x for x in result if x[key] == value]
if found:
return Result(data=found[0])
return Result(data=result)
async def get_job(identifier, job_id, **data) -> Result:
server_config = data.pop("server_config", {})
link = data.pop("link", None) or settings.SCHEDULER_SPREADSHEET
sheet = data.pop("sheet", None) or settings.SCHEDULER_SHEET_NAME
if server_config:
config = server_config
else:
config = await get_provider_sheet(link=link, sheet=sheet, provider=identifier)
if config:
instance = RPCService(config["host"], config["port"], config["server_method"])
jobs = instance.parse_jobs(job_id)
if jobs:
return Result(data=jobs[0])
return Result(error=f"Could not fetch info for job with id {job_id}")
async def clear_all_rows(link, sheet):
if not link or not sheet:
return Result(error="Missing `link` or `sheet` value")
instance = models.GoogleSheetInterface(**config)
instance.load_file(link, sheet)
instance.clear()
return await read_row(link, sheet, None, None)
async def add_to_sheet(link, sheet, value):
if not link or not sheet:
return Result(error="Missing `link` or `sheet` value")
instance = models.GoogleSheetInterface(**config)
instance.load_file(link, sheet)
key, value = instance.update_records(value)
return await read_row(link, sheet, key, value[key])
async def get_cloudinary_url(identifier, kind, public_id, **data) -> Result:
server_config = data.pop("server_config", {})
link = data.pop("link", None) or settings.MEDIA_SPREADSHEET
sheet = data.pop("sheet", None) or settings.MEDIA_SHEET_NAME
if server_config:
config = server_config
else:
config = await get_provider_sheet(link=link, sheet=sheet, provider=identifier)
if public_id and config:
instance = media_utils.MediaServiceAPI.get_instance(
resource_id=public_id, kind=kind, config=config
)
return Result(data=instance.build_url(secure=True, **data))
return Result(
error="Error getting the image in cloudinary, missing public_id or invalid config"
)
async def get_credentials(_id, link=None, sheet=None, credentials=None, **kwargs):
custom_credentials = {}
if credentials:
custom_credentials = credentials
provider_instance = await get_provider_sheet(link, sheet, _id, **custom_credentials)
if not provider_instance:
return Result(error="Could not find provider in link provided")
return provider_instance
async def add_multiple_rows(link, sheet, values):
if not link or not sheet:
return Result(error="Missing `link` or `sheet` value")
instance = models.GoogleSheetInterface(**config)
instance.load_file(link, sheet)
instance.bulk_add(values)
# for i in values:
# key, value = instance.update_records(i)
return await read_row(link, sheet, None, None)
async def get_email_content(provider, email_data=None, key="client", **kwargs):
generated_token, provider_instance = await get_refresh_token(
provider, key=key, with_provider_instance=True, **kwargs)
klass_options = {
"zoho": email_providers.ZohoProvider,
"gmail": email_providers.GmailProvider,
}
instance = klass_options[provider](generated_token, provider_instance)
try:
email_params = email_data or {}
data = await instance.get_email_content(**email_params)
return Result(data={
"content": data,
"refresh_token": generated_token["refresh_token"]
})
except Exception as e:
print(e)
return Result(error="Error fetching email content")
async def create_cloudinary_audio(identifier, **data) -> Result:
server_config = data.pop("server_config", {})
link = data.pop("link", None) or settings.MEDIA_SPREADSHEET
sheet = data.pop("sheet", None) or settings.MEDIA_SHEET_NAME
audio = data.pop("audio", None)
resource_type = data.pop("resource_type", None) or "raw"
if server_config:
config = server_config
else:
config = await get_provider_sheet(link=link, sheet=sheet, provider=identifier)
if audio and config:
result = media_utils.MediaServiceAPI.create_resource(
audio, config=config, resource_type=resource_type, **data
)
return Result(data=result)
return Result(
error="Error creating the audio in cloudinary, missing audio or invalid config"
)
async def delete_cloudinary_resource(identifier, public_id, **data) -> Result:
server_config = data.pop("server_config", {})
link = data.pop("link", None) or settings.MEDIA_SPREADSHEET
sheet = data.pop("sheet", None) or settings.MEDIA_SHEET_NAME
kind = data.pop("kind", None) or "image"
if server_config:
config = server_config
else:
config = await get_provider_sheet(link=link, sheet=sheet, provider=identifier)
if public_id and config:
instance = media_utils.MediaServiceAPI.get_instance(
resource_id=public_id, kind=kind, config=config
)
instance.delete()
return Result(data={"msg": "Successful"})
return Result(
error="Error deleting the resource in cloudinary, missing public_id or invalid config"
)
async def get_refresh_token(
provider,
link=None,
sheet=None,
refresh_token=None,
key="client",
credentials=None,
with_provider_instance=False,
**kwargs,
):
custom_credentials = {}
if credentials:
custom_credentials = credentials
provider_instance = await get_provider_sheet(link,
sheet,
provider,
key=key,
**custom_credentials)
if not provider_instance:
return Result(error="Could not find provider in link provided")
client_id = provider_instance.get("client_id")
if "client_id" in kwargs:
client_id = kwargs.pop("client_id")
if not client_id:
return Result(error="client_id not provided")
client_secret = provider_instance.get("client_secret")
if "client_secret" in kwargs:
client_secret = kwargs.pop("client_secret")
if not client_secret:
return Result(error="client_secret not provided")
if not refresh_token:
return Result(error="No refresh token passed")
_, new_token = oauth_implementation(
provider_instance,
client_id=client_id,
client_secret=client_secret,
token={"refresh_token": refresh_token},
**kwargs,
)
if with_provider_instance:
return new_token, provider_instance
return Result(data=new_token)
async def get_all_jobs(identifier, **data) -> Result:
server_config = data.pop("server_config", {})
link = data.pop("link", None) or settings.SCHEDULER_SPREADSHEET
sheet = data.pop("sheet", None) or settings.SCHEDULER_SHEET_NAME
status = data.get("status", "all")
if server_config:
config = server_config
else:
config = await get_provider_sheet(link=link, sheet=sheet, provider=identifier)
if config:
instance = RPCService(config["host"], config["port"], config["server_method"])
jobs = instance.parse_jobs()
if status == "paused":
jobs = [x for x in jobs if x["next_run_time"] == ""]
if status == "running":
jobs = [x for x in jobs if x["next_run_time"] != ""]
return Result(data=jobs)
return Result(error="Error with passed parameters")
async def get_emails(
provider,
search_config=None,
key="client",
**kwargs,
):
generated_token, provider_instance = await get_refresh_token(
provider, key=key, with_provider_instance=True, **kwargs)
klass_options = {
"zoho": email_providers.ZohoProvider,
"gmail": email_providers.GmailProvider,
}
instance = klass_options[provider](generated_token, provider_instance)
try:
emails = await instance.get_emails(search_config)
return Result(data={
"emails": emails,
"refresh_token": generated_token["refresh_token"]
})
except Exception as e:
print(e)
return Result(error="Error fetching emails")
async def web_spell_check_api(**data) -> Result:
config = data.pop("server_config", {})
spell_check_config = config.get("spell_check") or {}
url = spell_check_config.get("url")
text = data.get("text")
api_key = spell_check_config.get("api_key")
if url and text and api_key:
result = requests.get(
url,
params={
"text": text,
"lang": data.get("lang") or "en_US",
"format": "json",
"customerid": api_key,
"cmd": "check",
},
)
if result.status_code < 400:
response = result.json()
data = response["result"]
flat = [get_errors(x, text) for o in data for x in o["matches"]]
return Result(data=flat)
return Result(error="Error from processing check.")
return Result(error="Missing url, text or api_key")
async def create_cloudinary_image(identifier, **data) -> Result:
server_config = data.pop("server_config", {})
link = data.pop("link", None) or settings.MEDIA_SPREADSHEET
sheet = data.pop("sheet", None) or settings.MEDIA_SHEET_NAME
image = data.pop("image", None)
url = data.pop("url", None)
resource_type = data.pop("resource_type", None) or "image"
if server_config:
config = server_config
else:
print({"link": link, "sheet": sheet})
config = await get_provider_sheet(link=link, sheet=sheet, provider=identifier)
if url:
image = url
if image and config:
if url:
image = None
result = media_utils.MediaServiceAPI.create_resource(
image, config=config, resource_type=resource_type, url=url, **data
)
return Result(data=result)
return Result(
error="Error creating the image in cloudinary, missing image or invalid config"
)
async def get_access_and_refresh_token(provider,
link=None,
sheet=None,
key="client",
credentials=None,
**kwargs):
custom_credentials = {}
if credentials:
custom_credentials = credentials
provider_instance = await get_provider_sheet(link,
sheet,
provider,
key=key,
**custom_credentials)
if not provider_instance:
return Result(error="Could not find provider in link provided")
client_id = provider_instance.get("client_id")
if "client_id" in kwargs:
client_id = kwargs.pop("client_id")
if not client_id:
return Result(error="client_id not provided")
oauth_library = oauth_implementation(provider_instance,
client_id=client_id,
**kwargs)
if not oauth_library:
return Result(
error="Could not generate authorization url for provider passed")
authorization_response = kwargs.pop("authorization_response", None)
if not authorization_response:
return Result(error="No authorization_response passed")
client_secret = provider_instance.get("client_secret")
if "client_secret" in kwargs:
client_secret = kwargs.pop("client_secret")
if not client_secret:
return Result(error="client_secret not provided")
try:
token = resolve_token(
oauth_library,
provider_instance,
authorization_response=authorization_response,
client_secret=client_secret,
**kwargs,
)
return Result(data=token)
except CustomOAuth2Error as e:
return Result(error="Error when trying to get token")
async def google_nlp(**data):
content = data.get("text")
if not content:
return Result(error={"msg": "Missing text"})
client = language_v1.LanguageServiceClient()
encoding_type = language_v1.EncodingType.UTF8
document = language_v1.Document(content=content,
type_=language_v1.Document.Type.PLAIN_TEXT)
result = {}
response = client.analyze_entities(request={
"document": document,
"encoding_type": encoding_type
}, )
entities = []
for entity in response.entities:
sentiment = entity.sentiment
entities.append({
"name":
entity.name,
"type":
language_v1.Entity.Type(entity.type_).name,
"salience_score":
entity.salience,
"sentiment": {
"score": sentiment.score,
"magnitude": sentiment.magnitude,
},
"metadata": [{
"name": x[0],
"value": x[1]
} for x in entity.metadata.items()],
"mention": [{
"text": x.text.content,
"type": language_v1.EntityMention.Type(x.type_).name,
} for x in entity.mentions],
})
result["entities"] = entities
response = client.analyze_sentiment(request={
"document": document,
"encoding_type": encoding_type
})
sentiment = {
"score":
response.document_sentiment.score,
"magnitude":
response.document_sentiment.magnitude,
"sentences": [{
"text": x.text.content,
"score": x.sentiment.score,
"magnitude": x.sentiment.magnitude,
} for x in response.sentences],
}
result["sentiment"] = sentiment
response = client.analyze_syntax(request={
"document": document,
"encoding_type": encoding_type
})
tokens = []
for token in response.tokens:
text = token.text
part_of_speech = token.part_of_speech
dependency_edge = token.dependency_edge
tokens.append({
"text":
text.content,
"location":
text.begin_offset,
"part_of_speech":
language_v1.PartOfSpeech.Tag(part_of_speech.tag).name,
"voice":
language_v1.PartOfSpeech.Voice(part_of_speech.voice).name,
"tense":
language_v1.PartOfSpeech.Tense(part_of_speech.tense).name,
"lemma":
token.lemma,
"head_token_index":
dependency_edge.head_token_index,
"label":
language_v1.DependencyEdge.Label(dependency_edge.label).name,
})
result["tokens"] = tokens
response = client.classify_text(request={"document": document})
categories = []
for category in response.categories:
categories.append({
"name": category.name,
"confidence": category.confidence
})
result["categories"] = categories
# Detects the sentiment of the text
return Result(data=result)
async def edit_sheet(link, sheet, value):
if not link or not sheet:
return Result(error="Missing `link` or `sheet` value")
instance = models.GoogleSheetInterface(**config)
result = instance.edit_sheet(link, sheet, value)
return Result(data=result)